int main(int argc, char **argv) { skstream_t *in_stream; int in_rv = SKSTREAM_OK; int rv = SKSTREAM_OK; appSetup(argc, argv); /* never returns on error */ #error "Loop over files on command line or read from stdin." #error "Process each file, preferably in a separate function." /* For each input, process each record */ while (NULL != (in_stream = appNextInput(argc, argv))) { while ((in_rv = skStreamReadRecord(in_stream, &rwrec))==SKSTREAM_OK) { /* process record */ rv = skStreamWriteRecord(out_stream, &rwrec); if (SKSTREAM_OK != rv) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); skStreamDestroy(&in_stream); goto END; } } if (SKSTREAM_ERR_EOF != in_rv) { skStreamPrintLastErr(in_stream, in_rv, &skAppPrintErr); } skStreamDestroy(&in_stream); } rv = skStreamClose(out_stream); if (SKSTREAM_OK != rv) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); } END: return ((SKSTREAM_OK == rv) ? EXIT_SUCCESS : EXIT_FAILURE); }
/* * stream = appNextInput(argc, argv); * * Open and return the next input file from the command line or the * standard input if no files were given on the command line. */ static skstream_t * appNextInput( int argc, char **argv) { static int initialized = 0; skstream_t *stream = NULL; const char *fname = NULL; int rv; if (arg_index < argc) { /* get current file and prepare to get next */ fname = argv[arg_index]; ++arg_index; } else { if (initialized) { /* no more input */ return NULL; } /* input is from stdin */ fname = "stdin"; } initialized = 1; /* create stream and open file */ rv = skStreamOpenSilkFlow(&stream, fname, SK_IO_READ); if (rv) { skStreamPrintLastErr(stream, rv, NULL); skStreamDestroy(&stream); } return stream; }
/* * appTeardown() * * Teardown all modules, close all files, and tidy up all * application state. * * This function is idempotent. */ static void appTeardown( void) { static int teardownFlag = 0; int rv; if (teardownFlag) { return; } teardownFlag = 1; /* close SiLK flow output file */ if (silk_output) { rv = skStreamClose(silk_output); if (rv && rv != SKSTREAM_ERR_NOT_OPEN) { skStreamPrintLastErr(silk_output, rv, &skAppPrintErr); } skStreamDestroy(&silk_output); } skpcTeardown(); /* set level to "warning" to avoid the "Stopped logging" message */ sklogSetLevel("warning"); sklogTeardown(); skOptionsNotesTeardown(); skOptionsCtxDestroy(&optctx); skAppUnregister(); }
/* * status = appOptionsHandler(cData, opt_index, opt_arg); * * Called by skOptionsParse(), this handles a user-specified switch * that the application has registered, typically by setting global * variables. Returns 1 if the switch processing failed or 0 if it * succeeded. Returning a non-zero from from the handler causes * skOptionsParse() to return a negative value. * * The clientData in 'cData' is typically ignored; 'opt_index' is * the index number that was specified as the last value for each * struct option in appOptions[]; 'opt_arg' is the user's argument * to the switch for options that have a REQUIRED_ARG or an * OPTIONAL_ARG. */ static int appOptionsHandler( clientData UNUSED(cData), int opt_index, char *opt_arg) { int rv; switch ((appOptionsEnum)opt_index) { #if 0 case OPT_HELP_FIELDS: helpFields(USAGE_FH); exit(EXIT_SUCCESS); #endif /* #if 0 */ case OPT_KEYS: if (keys_arg) { skAppPrintErr("Invalid %s: Switch used multiple times", appOptions[opt_index].name); return 1; } keys_arg = opt_arg; break; case OPT_COUNTERS: if (counters_arg) { skAppPrintErr("Invalid %s: Switch used multiple times", appOptions[opt_index].name); return 1; } counters_arg = opt_arg; break; case OPT_OUTPUT_PATH: if (output) { skAppPrintErr("Invalid %s: Switch used multiple times", appOptions[opt_index].name); return 1; } if ((rv = skStreamCreate(&output, SK_IO_WRITE, SK_CONTENT_SILK)) || (rv = skStreamBind(output, opt_arg))) { skStreamPrintLastErr(output, rv, &skAppPrintErr); skStreamDestroy(&output); return 1; } break; } return 0; /* OK */ }
int main(int argc, char **argv) { skstream_t *inputFile; skPrefixMap_t *prefixMap; skPrefixMapErr_t map_error = SKPREFIXMAP_OK; char buf[DICTIONARY_ENTRY_BUFLEN]; int rv; appSetup(argc, argv); /* never returns on error */ /* Okay. Now we should open the prefixmap file, read it in, and */ /* then look up our address! */ if ((rv = skStreamCreate(&inputFile, SK_IO_READ, SK_CONTENT_SILK)) || (rv = skStreamBind(inputFile, prefixmap_test_opt.map_file)) || (rv = skStreamOpen(inputFile))) { skStreamPrintLastErr(inputFile, rv, &skAppPrintErr); skStreamDestroy(&inputFile); exit(EXIT_FAILURE); } map_error = skPrefixMapRead(&prefixMap, inputFile); skStreamDestroy(&inputFile); if ( SKPREFIXMAP_OK != map_error ) { skAppPrintErr("Failed to read map file: %s", skPrefixMapStrerror(map_error)); exit(EXIT_FAILURE); } if ( prefixmap_test_opt.string ) { int v = skPrefixMapFindString(prefixMap, &prefixmap_test_opt.address, buf, sizeof(buf)); if ( v < 0 ) { strncpy(buf, "(null)", sizeof(buf)); } printf("%s\n", buf); } else { printf("%d\n", skPrefixMapFindValue(prefixMap, &prefixmap_test_opt.address)); } skPrefixMapDelete(prefixMap); /* done */ appTeardown(); return 0; }
int main(int argc, char **argv) { skstream_t *stream = NULL; ssize_t rv; /* Global setup */ appSetup(argc, argv); /* process input */ while ((rv = skOptionsCtxNextSilkFile(optctx, &stream, &skAppPrintErr)) == 0) { skStreamSetIPv6Policy(stream, ipv6_policy); if (0 != processFile(stream)) { skAppPrintErr("Error processing input from %s", skStreamGetPathname(stream)); skStreamDestroy(&stream); return EXIT_FAILURE; } skStreamDestroy(&stream); } if (rv < 0) { exit(EXIT_FAILURE); } rv = skAggBagWrite(ab, output); if (rv) { if (SKAGGBAG_E_WRITE == rv) { skStreamPrintLastErr(output, skStreamGetLastReturnValue(output), &skAppPrintErr); } else { skAppPrintErr("Error writing Aggregate Bag to '%s': %s", skStreamGetPathname(output), skAggBagStrerror(rv)); } exit(EXIT_FAILURE); } skAggBagDestroy(&ab); /* Done, do cleanup */ appTeardown(); return 0; }
/* * status = tailFile(stream); * * Read SiLK flow records from the file at 'stream' and store the * most recent 'tail_recs' number of records in the 'tail_buf' * buffer. * * Return -1 on read error, or 0 otherwise. */ static int tailFile( skstream_t *stream) { int rv = SKSTREAM_OK; while ((rv = skStreamReadRecord(stream, tail_buf_cur)) == SKSTREAM_OK) { ++tail_buf_cur; if (tail_buf_cur == &tail_buf[tail_recs]) { tail_buf_cur = tail_buf; tail_buf_full = 1; } } if (SKSTREAM_ERR_EOF != rv) { skStreamPrintLastErr(stream, rv, &skAppPrintErr); return -1; } return 0; }
int main(int argc, char **argv) { int rv; appSetup(argc, argv); /* never returns on error */ sortRandom(); /* close the file */ if ((rv = skStreamClose(out_rwios)) || (rv = skStreamDestroy(&out_rwios))) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); appExit(EXIT_FAILURE); } out_rwios = NULL; appExit(EXIT_SUCCESS); return 0; /* NOTREACHED */ }
char * get_ipa_config( void) { skstream_t *conf_stream = NULL; char filename[PATH_MAX]; char line[IPA_CONFIG_LINE_LENGTH]; char *ipa_url = NULL; int rv; /* Read in the data file */ if (NULL == skFindFile(IPA_CONFIG_FILE, filename, sizeof(filename), 1)) { skAppPrintErr("Could not locate config file '%s'.", IPA_CONFIG_FILE); return NULL; } /* open input */ if ((rv = skStreamCreate(&conf_stream, SK_IO_READ, SK_CONTENT_TEXT)) || (rv = skStreamBind(conf_stream, filename)) || (rv = skStreamSetCommentStart(conf_stream, "#")) || (rv = skStreamOpen(conf_stream))) { skStreamPrintLastErr(conf_stream, rv, &skAppPrintErr); skStreamDestroy(&conf_stream); exit(EXIT_FAILURE); } while (skStreamGetLine(conf_stream, line, sizeof(line), NULL) == SKSTREAM_OK) { /* FIXME: smarter config file reading, please */ if (strlen(line) > 0) { ipa_url = strdup(line); break; } } skStreamDestroy(&conf_stream); /* Should be free()d by the caller */ return ipa_url; }
/* * count = pdu2silk(filename); * * Read PDUs from 'filename' and write records to the global * 'silk_output' file. Return number of records processed, or -1 * on error. */ static int64_t pdu2silk( const char *filename) { static unsigned int file_count = 0; char probe_name[128]; skPDUSource_t *pdu_src; skFlowSourceParams_t params; int64_t count; rwRec rwrec; int rv; ++file_count; snprintf(probe_name, sizeof(probe_name), "input%04u", file_count); params.path_name = filename; skpcProbeSetName(probe, probe_name); pdu_src = skPDUSourceCreate(probe, ¶ms); if (pdu_src == NULL) { return -1; } count = 0; while (-1 != skPDUSourceGetGeneric(pdu_src, &rwrec)) { ++count; rv = skStreamWriteRecord(silk_output, &rwrec); if (rv) { skStreamPrintLastErr(silk_output, rv, &skAppPrintErr); if (SKSTREAM_ERROR_IS_FATAL(rv)) { exit(EXIT_FAILURE); } } } skPDUSourceLogStatsAndClear(pdu_src); skPDUSourceDestroy(pdu_src); return count; }
int main(int argc, char **argv) { int rv; appSetup(argc, argv); /* read input file */ if (ip_ranges) { if (buildIPSetRanges(in_stream)) { return 1; } } else { if (buildIPSetWildcards(in_stream)) { return 1; } } skIPSetClean(ipset); /* write output to stream */ rv = skIPSetWrite(ipset, out_stream); if (rv) { if (SKIPSET_ERR_FILEIO == rv) { skStreamPrintLastErr(out_stream, skStreamGetLastReturnValue(out_stream), &skAppPrintErr); } else { skAppPrintErr("Unable to write IPset to '%s': %s", skStreamGetPathname(out_stream),skIPSetStrerror(rv)); } return 1; } skStreamDestroy(&in_stream); skStreamDestroy(&out_stream); /* done */ return 0; }
/* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup( int argc, char **argv) { SILK_FEATURES_DEFINE_STRUCT(features); unsigned int optctx_flags; sk_file_header_t *silk_hdr; int logmask; int rv; /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skAppVerifyFeatures(&features, NULL); skOptionsSetUsageCallback(&appUsageLong); optctx_flags = (SK_OPTIONS_CTX_INPUT_BINARY | SK_OPTIONS_CTX_XARGS); /* register the options */ if (skOptionsCtxCreate(&optctx, optctx_flags) || skOptionsCtxOptionsRegister(optctx) || skOptionsRegister(appOptions, &appOptionsHandler, NULL) || skOptionsNotesRegister(NULL) || skCompMethodOptionsRegister(&comp_method)) { skAppPrintErr("Unable to register options"); exit(EXIT_FAILURE); } /* enable the logger */ sklogSetup(0); sklogSetDestination("stderr"); sklogSetStampFunction(&logprefix); /* register the teardown handler */ if (atexit(appTeardown) < 0) { skAppPrintErr("Unable to register appTeardown() with atexit()"); appTeardown(); exit(EXIT_FAILURE); } /* parse the options */ rv = skOptionsCtxOptionsParse(optctx, argc, argv); if (rv < 0) { skAppUsage(); /* never returns */ } if ('\0' == log_destination[0]) { strncpy(log_destination, LOG_DESTINATION_DEFAULT, sizeof(log_destination)); } else { sklogSetLevel("debug"); } sklogSetDestination(log_destination); /* default output is "stdout" */ if (!silk_output) { if ((rv =skStreamCreate(&silk_output,SK_IO_WRITE,SK_CONTENT_SILK_FLOW)) || (rv = skStreamBind(silk_output, "-"))) { skStreamPrintLastErr(silk_output, rv, &skAppPrintErr); exit(EXIT_FAILURE); } } /* get the header */ silk_hdr = skStreamGetSilkHeader(silk_output); /* open the output */ if ((rv = skHeaderSetCompressionMethod(silk_hdr, comp_method)) || (rv = skOptionsNotesAddToStream(silk_output)) || (rv = skHeaderAddInvocation(silk_hdr, 1, argc, argv)) || (rv = skStreamOpen(silk_output)) || (rv = skStreamWriteSilkHeader(silk_output))) { skStreamPrintLastErr(silk_output, rv, &skAppPrintErr); exit(EXIT_FAILURE); } if (skpcSetup()) { exit(EXIT_FAILURE); } if (skpcProbeCreate(&probe, PROBE_ENUM_NETFLOW_V5)) { exit(EXIT_FAILURE); } skpcProbeSetName(probe, skAppName()); skpcProbeSetFileSource(probe, "/dev/null"); if (parseLogFlags(log_flags)) { exit(EXIT_FAILURE); } if (skpcProbeVerify(probe, 0)) { exit(EXIT_FAILURE); } /* set level to "warning" to avoid the "Started logging" message */ logmask = sklogGetMask(); sklogSetLevel("warning"); sklogOpen(); sklogSetMask(logmask); return; /* OK */ }
static int compareFiles( char **file) { skstream_t *stream[2] = {NULL, NULL}; rwRec rec[2]; int i; int rv; int status = 2; uint64_t rec_count = 0; int eof = -1; memset(stream, 0, sizeof(stream)); memset(rec, 0, sizeof(rec)); for (i = 0; i < 2; ++i) { if ((rv = skStreamCreate(&stream[i], SK_IO_READ, SK_CONTENT_SILK_FLOW)) || (rv = skStreamBind(stream[i], file[i])) || (rv = skStreamOpen(stream[i])) || (rv = skStreamReadSilkHeader(stream[i], NULL))) { /* Give up if we can't read the beginning of the silk header */ if (rv != SKSTREAM_OK) { if (!quiet) { skStreamPrintLastErr(stream[i], rv, &skAppPrintErr); } goto END; } } } while ((rv = skStreamReadRecord(stream[0], &rec[0])) == SKSTREAM_OK) { rv = skStreamReadRecord(stream[1], &rec[1]); if (rv != SKSTREAM_OK) { if (rv == SKSTREAM_ERR_EOF) { /* file 0 longer than file 1 */ status = 1; eof = 1; } else { if (!quiet) { skStreamPrintLastErr(stream[1], rv, &skAppPrintErr); } status = -1; } goto END; } ++rec_count; if (0 != memcmp(&rec[0], &rec[1], sizeof(rwRec))) { status = 1; goto END; } } if (rv != SKSTREAM_ERR_EOF) { if (!quiet) { skStreamPrintLastErr(stream[0], rv, &skAppPrintErr); } } else { rv = skStreamReadRecord(stream[1], &rec[1]); switch (rv) { case SKSTREAM_OK: /* file 1 longer than file 0 */ status = 1; eof = 0; break; case SKSTREAM_ERR_EOF: /* files identical */ status = 0; break; default: if (!quiet) { skStreamPrintLastErr(stream[1], rv, &skAppPrintErr); } break; } } END: for (i = 0; i < 2; ++i) { skStreamDestroy(&stream[i]); } if (1 == status && !quiet) { if (eof != -1) { printf("%s %s differ: EOF %s\n", file[0], file[1], file[eof]); } else { printf(("%s %s differ: record %" PRIu64 "\n"), file[0], file[1], rec_count); #ifdef RWCOMPARE_VERBOSE printRecords(rec); #endif } } return status; }
/* * Helper function that opens the flowcap file at 'path'. The * handle to the file is put in the location specified by 'stream'. * The name of the probe is read from the file's header, and the * probe object is put into 'probe'. Returns 0 on success, or -1 * on the following error conditions: * * -- 'path' is NULL. * -- unable to open file * -- file is not a valid SiLK file * -- file does not contain a Probename Header * -- the probe_name does not map to a valid probe in probeconf * * NOTES: * * Flowcap V16 files have the probe name in the header. * * Flowcap Files V2-V5 have the sensor name and probe name in the * header. When these are processed in SiLK 1.0, these get mapped * to the probe name "<sensor>_<probe>". * * Flowcap Files V1 have no probe information and are no longer * supported. */ static int flowcapSourceCreateFromFile( const char *path, skstream_t **stream, skpc_probe_t **probe) { sk_file_header_t *hdr; sk_header_entry_t *sp_hdr; const char *probe_name; int rv; /* Argument checking */ if (path == NULL) { ERRMSG("NULL path passed to flowcapSourceCreateFromFile."); return -1; } /* Valid file checking */ rv = skStreamOpenSilkFlow(stream, path, SK_IO_READ); if (rv) { CRITMSG("Unable to open '%s' for reading.", path); skStreamPrintLastErr(*stream, rv, &ERRMSG); goto error; } /* * File should have a Probename header * * Flowcap V16 files have the probe name in the header. * * Flowcap Files V2-V5 have a separate sensor name and probe name * in the header. In SiLK 1.0, these get mapped to the single * probe name "<sensor>_<probe>". * * Flowcap Files V1 have no probe information and are not * supported. */ hdr = skStreamGetSilkHeader(*stream); sp_hdr = skHeaderGetFirstMatch(hdr, SK_HENTRY_PROBENAME_ID); if (sp_hdr == NULL) { CRITMSG("No probename header in %s.", path); goto error; } probe_name = skHentryProbenameGetProbeName(sp_hdr); if (probe_name == NULL || probe_name[0] == '\0') { CRITMSG("Unable to get probename from flowcap file '%s'.", path); goto error; } /* Use the probe_name to find the skpc_probe_t object. */ *probe = (skpc_probe_t*)skpcProbeLookupByName(probe_name); if (*probe == NULL) { CRITMSG("The sensor configuration file does not define probe '%s'", probe_name); goto error; } /* Verify that the probe has sensors associated with it */ if (skpcProbeGetSensorCount(*probe) == 0) { CRITMSG("Probe '%s' is not associated with a sensor", probe_name); goto error; } /* File has been validated. We're done. */ return 0; error: skStreamDestroy(stream); return -1; }
/* * status = readerGetRecord(&out_rwrec, &out_probe, flow_processor); * * Invoked by input_mode_type->get_record_fn(); */ static fp_get_record_result_t readerGetRecord( rwRec *out_rwrec, const skpc_probe_t **out_probe, flow_proc_t *fproc) { static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; skstream_t *fcfile; const char *filename; fp_get_record_result_t retVal = FP_GET_ERROR; int rv; pthread_mutex_lock(&mutex); /* If we don't have a source, get a file from the directory poller * and start processing it. */ if (fproc->flow_src == NULL) { switch (readerGetNextValidFile(fproc)) { case 0: /* Success */ break; case -1: /* Error getting file name (maybe in shutdown?) */ goto END; case -2: /* Error opening file */ default: /* Unexpected value */ retVal = FP_FATAL_ERROR; goto END; } } fcfile = (skstream_t*)fproc->flow_src; /* Assume we can get a record from the probe. */ retVal = FP_RECORD; *out_probe = fproc->probe; /* Try to get a record */ rv = skStreamReadRecord(fcfile, out_rwrec); if (rv) { /* get failed: either at EOF or got an error. */ if (rv != SKSTREAM_ERR_EOF) { skStreamPrintLastErr(fcfile, rv, &WARNINGMSG); } retVal = FP_FILE_BREAK; *out_probe = NULL; /* Print results for the file we just processed. */ filename = skStreamGetPathname(fcfile); INFOMSG("Processed file %s, %" PRIu64 " records.", filename, skStreamGetRecordCount(fcfile)); skStreamClose(fcfile); /* Either archive the file or remove it */ archiveDirectoryInsertOrRemove(filename, NULL); /* All done with the flow source */ skStreamDestroy(&fcfile); fproc->flow_src = NULL; fproc->probe = NULL; } END: pthread_mutex_unlock(&mutex); return retVal; }
/* * skplugin_err_t pmapfile_handler(const char *opt_arg, void *cbdata) * * Handler for --pmap-file option. Actually registers the filter and * fields. */ static skplugin_err_t pmapfile_handler( const char *opt_arg, void UNUSED(*cbdata)) { /* Whether we have seen any unnamed pmaps */ static int have_unnamed_pmap = 0; skPrefixMapErr_t map_error = SKPREFIXMAP_OK; skstream_t *stream = NULL; skPrefixMap_t *prefix_map = NULL; pmap_data_t *pmap_data = NULL; int ok; const char *filename; const char *sep; const char *mapname; char *prefixed_name = NULL; char *short_prefixed_name; size_t namelen = 0; size_t i; int rv = SKPLUGIN_ERR; skplugin_callbacks_t regdata; /* We can only have one pmap whenever we any any pmap without a * mapname. If we've seen one and enter this function a second * time, it is an error */ if (have_unnamed_pmap) { skAppPrintErr(("Invalid %s: You may use only one prefix map" " when you are\n" "\tusing a prefix map without specifying a mapname"), pmap_file_option); return SKPLUGIN_ERR; } /* Parse the argument into a field name and file name */ sep = strchr(opt_arg, ':'); if (NULL == sep) { /* We do not have a mapname. We'll check for one in the pmap * once we read it. */ mapname = NULL; filename = opt_arg; } else { /* A mapname was supplied on the command line */ if (sep == opt_arg) { skAppPrintErr("Invalid %s: Zero length mapnames are not allowed", pmap_file_option); return SKPLUGIN_ERR; } if (memchr(opt_arg, ',', sep - opt_arg) != NULL) { skAppPrintErr("Invalid %s: The mapname may not include a comma", pmap_file_option); return SKPLUGIN_ERR; } mapname = opt_arg; filename = sep + 1; namelen = sep - opt_arg; } ok = skpinOpenDataInputStream(&stream, SK_CONTENT_SILK, filename); if (ok == -1) { /* problem opening file */ skAppPrintErr("Failed to open the prefix map file '%s'", filename); return SKPLUGIN_ERR; } if (ok == 1) { /* master needs to process the file, since it may contain the * map name we use for creating switches */ if ((rv = skStreamCreate(&stream, SK_IO_READ, SK_CONTENT_SILK)) || (rv = skStreamBind(stream, filename)) || (rv = skStreamOpen(stream))) { skStreamPrintLastErr(stream, rv, &skAppPrintErr); skStreamDestroy(&stream); return SKPLUGIN_ERR; } /* the master can ignore the file for filtering */ ignore_prefix_map = 1; } map_error = skPrefixMapRead(&prefix_map, stream); if (SKPREFIXMAP_OK != map_error) { if (SKPREFIXMAP_ERR_IO == map_error) { skStreamPrintLastErr(stream, skStreamGetLastReturnValue(stream), &skAppPrintErr); } else { skAppPrintErr("Failed to read the prefix map file '%s': %s", opt_arg, skPrefixMapStrerror(map_error)); } skStreamDestroy(&stream); return SKPLUGIN_ERR; } skStreamDestroy(&stream); if (NULL == mapname) { /* No mapname was supplied on the command line. Check for a * mapname insided the pmap. */ mapname = skPrefixMapGetMapName(prefix_map); if (mapname) { /* The pmap supplied a mapname */ namelen = strlen(mapname); } else { /* No mapname. Accept for legacy purposes, unless we have * read any other pmaps */ have_unnamed_pmap = 1; if (skVectorGetCount(pmap_vector) != 0) { skAppPrintErr(("Invalid %s: You may use only one prefix map" " when you are\n" "\t using a prefix map without" " specifying a mapname"), pmap_file_option); goto END; } } } /* Allocate the pmap_data structure */ pmap_data = (pmap_data_t *)calloc(1, sizeof(*pmap_data)); if (pmap_data == NULL) { ERR_NO_MEM(pmap_data); rv = SKPLUGIN_ERR_FATAL; goto END; } /* pmap_data now "owns" prefix_map */ pmap_data->pmap = prefix_map; prefix_map = NULL; /* Cache the content type */ pmap_data->type = skPrefixMapGetContentType(pmap_data->pmap); /* Fill the direction structure for each direction */ pmap_data->sdir.dir = DIR_SOURCE; pmap_data->ddir.dir = DIR_DEST; pmap_data->adir.dir = DIR_ANY; pmap_data->sdir.data = pmap_data; pmap_data->ddir.data = pmap_data; pmap_data->adir.data = pmap_data; /* Record the path to the pmap file */ pmap_data->filepath = strdup(filename); if (NULL == pmap_data->filepath) { ERR_NO_MEM(pmap_data->filepath); rv = SKPLUGIN_ERR_FATAL; goto END; } if (mapname == NULL) { /* Pmap without a mapname. */ /* Add the proper legacy option names to the pmap_data structure */ switch (pmap_data->type) { case SKPREFIXMAP_CONT_ADDR_V4: case SKPREFIXMAP_CONT_ADDR_V6: pmap_data->sdir.filter_option = strdup(pmap_saddress_option); pmap_data->ddir.filter_option = strdup(pmap_daddress_option); pmap_data->adir.filter_option = strdup(pmap_aaddress_option); break; case SKPREFIXMAP_CONT_PROTO_PORT: pmap_data->sdir.filter_option = strdup(pmap_sport_proto_option); pmap_data->ddir.filter_option = strdup(pmap_dport_proto_option); pmap_data->adir.filter_option = strdup(pmap_aport_proto_option); break; } if ((pmap_data->sdir.filter_option == NULL) || (pmap_data->ddir.filter_option == NULL) || (pmap_data->adir.filter_option == NULL)) { ERR_NO_MEM(filter_option); rv = SKPLUGIN_ERR_FATAL; goto END; } pmap_data->mapname = strdup(pmap_title_val); pmap_data->sdir.field_name = strdup(pmap_title_sval); pmap_data->ddir.field_name = strdup(pmap_title_dval); if ((pmap_data->mapname == NULL) || (pmap_data->sdir.field_name == NULL) || (pmap_data->ddir.field_name == NULL)) { ERR_NO_MEM(field_name); rv = SKPLUGIN_ERR_FATAL; goto END; } } else { /* if (mapname == NULL) */ /* Create the field names*/ pmap_data->mapname = (char*)malloc(namelen + 1); if (NULL == pmap_data->mapname) { ERR_NO_MEM(pmap_data->mapname); rv = SKPLUGIN_ERR_FATAL; goto END; } strncpy(pmap_data->mapname, mapname, namelen); pmap_data->mapname[namelen] = '\0'; /* Allocate space for the [pmap-]{src-,dst-}<mapname> string */ prefixed_name = (char*)malloc(namelen + pmap_prefix_len + dir_name_len + 1); if (NULL == prefixed_name) { ERR_NO_MEM(prefixed_name); rv = SKPLUGIN_ERR_FATAL; goto END; } /* Copy in the pmap- prefix */ strncpy(prefixed_name, pmap_prefix, pmap_prefix_len); /* short name (for fields) starts at the {src-,dst-} */ short_prefixed_name = prefixed_name + pmap_prefix_len; /* add in the actual field name, and zero terminate it */ strncpy(short_prefixed_name + dir_name_len, mapname, namelen); short_prefixed_name[namelen + dir_name_len] = '\0'; /* Create the destination-themed names */ strncpy(short_prefixed_name, src_dir_name, dir_name_len); pmap_data->sdir.filter_option = strdup(prefixed_name); pmap_data->sdir.field_name = strdup(short_prefixed_name); if ((pmap_data->sdir.filter_option == NULL) || (pmap_data->sdir.field_name == NULL)) { ERR_NO_MEM(pmap_data->sdir); rv = SKPLUGIN_ERR_FATAL; goto END; } strncpy(short_prefixed_name, dst_dir_name, dir_name_len); pmap_data->ddir.filter_option = strdup(prefixed_name); pmap_data->ddir.field_name = strdup(short_prefixed_name); if ((pmap_data->ddir.filter_option == NULL) || (pmap_data->ddir.field_name == NULL)) { ERR_NO_MEM(pmap_data->ddir); rv = SKPLUGIN_ERR_FATAL; goto END; } strncpy(short_prefixed_name, any_dir_name, dir_name_len); pmap_data->adir.filter_option = strdup(prefixed_name); if (pmap_data->adir.filter_option == NULL) { ERR_NO_MEM(pmap_data->adir); rv = SKPLUGIN_ERR_FATAL; goto END; } /* Free the temporary name buffer */ free(prefixed_name); prefixed_name = NULL; } /* if (mapname == NULL) */ /* Verify unique field names */ for (i = 0; i < skVectorGetCount(pmap_vector); i++) { pmap_data_t *p; skVectorGetValue(&p, pmap_vector, i); if ((strcmp(pmap_data->mapname, p->mapname) == 0) || (strcmp(pmap_data->sdir.field_name, p->sdir.field_name) == 0) || (strcmp(pmap_data->ddir.field_name, p->ddir.field_name) == 0)) { skAppPrintErr(("Invalid %s: Multiple pmaps use the mapname '%s':\n" "\t%s\n\t%s"), pmap_file_option, pmap_data->mapname, p->filepath, pmap_data->filepath); rv = SKPLUGIN_ERR; goto END; } } /* Register fields and filter options */ memset(®data, 0, sizeof(regdata)); regdata.init = pmap_field_init; regdata.column_width = 0; regdata.bin_bytes = 4; regdata.rec_to_text = pmap_text_fn; regdata.rec_to_bin = pmap_bin_fn; regdata.bin_to_text = pmap_bin_to_text_fn; for (i = 0; i < 2; ++i) { directed_pmap_data_t *dir = ((0 == i) ? &pmap_data->sdir : &pmap_data->ddir); skpinRegField(&dir->field, dir->field_name, NULL, ®data, dir); skpinRegOption2(dir->filter_option, REQUIRED_ARG, NULL, &pmap_filter_help, &pmap_handle_filter_option, dir, 1, SKPLUGIN_FN_FILTER); } /* Register the "any" filter separately */ skpinRegOption2(pmap_data->adir.filter_option, REQUIRED_ARG, NULL, &pmap_filter_help, &pmap_handle_filter_option, &pmap_data->adir, 1, SKPLUGIN_FN_FILTER); if (skVectorAppendValue(pmap_vector, &pmap_data)) { rv = SKPLUGIN_ERR_FATAL; goto END; } rv = SKPLUGIN_OK; END: /* Free the temporary name buffer */ if (prefixed_name) { free(prefixed_name); } if (rv != SKPLUGIN_OK) { if (prefix_map) { skPrefixMapDelete(prefix_map); } if (pmap_data) { pmap_data_destroy(pmap_data); } } return (skplugin_err_t)rv; }
/* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup( int argc, char **argv) { SILK_FEATURES_DEFINE_STRUCT(features); int arg_index; int rv; /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skAppVerifyFeatures(&features, NULL); skOptionsSetUsageCallback(&appUsageLong); /* register the options */ if (skOptionsRegister(appOptions, &appOptionsHandler, NULL)) { skAppPrintErr("Unable to register options"); exit(EXIT_FAILURE); } /* register the teardown hanlder */ if (atexit(appTeardown) < 0) { skAppPrintErr("Unable to register appTeardown() with atexit()"); appTeardown(); exit(EXIT_FAILURE); } /* parse the options */ arg_index = skOptionsParse(argc, argv); if (arg_index < 0) { /* options parsing should print error */ skAppUsage(); /* never returns */ } /* check for extraneous arguments */ if (arg_index != argc) { skAppPrintErr("Too many arguments or unrecognized switch '%s'", argv[arg_index]); skAppUsage(); /* never returns */ } /* check for one and only one of --input-file and --address */ if (NULL != app_opt.input_file) { if (NULL != app_opt.address) { skAppPrintErr("Only one of --%s or --%s may be specified.", appOptions[OPT_ADDRESS].name, appOptions[OPT_INPUT_FILE].name); skAppUsage(); } } else if (NULL == app_opt.address) { skAppPrintErr("Either the --%s or --%s option is required.", appOptions[OPT_ADDRESS].name, appOptions[OPT_INPUT_FILE].name); skAppUsage(); } /* find and load the map file */ if (skCountrySetup(app_opt.map_file, &skAppPrintErr)) { exit(EXIT_FAILURE); } /* use default for print-ips if unset by user */ if (-1 == app_opt.print_ips) { if (app_opt.input_file) { app_opt.print_ips = 1; } else { app_opt.print_ips = 0; } } /* create the output stream. Only page output when processing * input file */ if ((rv = skStreamCreate(&out, SK_IO_WRITE, SK_CONTENT_TEXT)) || (rv = skStreamBind(out, app_opt.output_path))) { skStreamPrintLastErr(out, rv, &skAppPrintErr); exit(EXIT_FAILURE); } if (app_opt.input_file) { rv = skStreamPageOutput(out, app_opt.pager); if (rv) { skStreamPrintLastErr(out, rv, &skAppPrintErr); exit(EXIT_FAILURE); } } rv = skStreamOpen(out); if (rv) { skStreamPrintLastErr(out, rv, &skAppPrintErr); exit(EXIT_FAILURE); } return; /* OK */ }
/* * Process a single input stream (file) of SiLK Flow records: Copy * the header entries from the input stream to the output stream, * read the file, fill a Key and Counter for each flow record, and * add the Key and Counter to the AggBag. */ static int processFile( skstream_t *stream) { sk_aggbag_field_t k_it; sk_aggbag_field_t c_it; sk_aggbag_aggregate_t key; sk_aggbag_aggregate_t counter; skipaddr_t ip; rwRec rwrec; ssize_t rv; ssize_t err; /* copy invocation and notes (annotations) from the SiLK Flow * files to the output stream; these headers will not be written * to the output if --invocation-strip or --notes-strip was * specified. */ rv = skHeaderCopyEntries(skStreamGetSilkHeader(output), skStreamGetSilkHeader(stream), SK_HENTRY_INVOCATION_ID); if (rv) { skStreamPrintLastErr(output, rv, &skAppPrintErr); } rv = skHeaderCopyEntries(skStreamGetSilkHeader(output), skStreamGetSilkHeader(stream), SK_HENTRY_ANNOTATION_ID); if (rv) { skStreamPrintLastErr(output, rv, &skAppPrintErr); } err = SKAGGBAG_OK; while (SKSTREAM_OK == (rv = skStreamReadRecord(stream, &rwrec))) { skAggBagInitializeKey(ab, &key, &k_it); do { switch (skAggBagFieldIterGetType(&k_it)) { case SKAGGBAG_FIELD_SIPv6: case SKAGGBAG_FIELD_SIPv4: rwRecMemGetSIP(&rwrec, &ip); skAggBagAggregateSetIPAddress(&key, &k_it, &ip); break; case SKAGGBAG_FIELD_DIPv6: case SKAGGBAG_FIELD_DIPv4: rwRecMemGetDIP(&rwrec, &ip); skAggBagAggregateSetIPAddress(&key, &k_it, &ip); break; case SKAGGBAG_FIELD_NHIPv6: case SKAGGBAG_FIELD_NHIPv4: rwRecMemGetNhIP(&rwrec, &ip); skAggBagAggregateSetIPAddress(&key, &k_it, &ip); break; case SKAGGBAG_FIELD_SPORT: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetSPort(&rwrec)); break; case SKAGGBAG_FIELD_DPORT: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetDPort(&rwrec)); break; case SKAGGBAG_FIELD_ICMP_TYPE: skAggBagAggregateSetUnsigned( &key, &k_it, (rwRecIsICMP(&rwrec) ? rwRecGetIcmpType(&rwrec) : 0)); break; case SKAGGBAG_FIELD_ICMP_CODE: skAggBagAggregateSetUnsigned( &key, &k_it, (rwRecIsICMP(&rwrec) ? rwRecGetIcmpCode(&rwrec) : 0)); break; case SKAGGBAG_FIELD_PROTO: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetProto(&rwrec)); break; case SKAGGBAG_FIELD_PACKETS: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetPkts(&rwrec)); break; case SKAGGBAG_FIELD_BYTES: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetBytes(&rwrec)); break; case SKAGGBAG_FIELD_FLAGS: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetFlags(&rwrec)); break; case SKAGGBAG_FIELD_SID: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetSensor(&rwrec)); break; case SKAGGBAG_FIELD_INPUT: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetInput(&rwrec)); break; case SKAGGBAG_FIELD_OUTPUT: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetOutput(&rwrec)); break; case SKAGGBAG_FIELD_INIT_FLAGS: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetInitFlags(&rwrec)); break; case SKAGGBAG_FIELD_REST_FLAGS: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetRestFlags(&rwrec)); break; case SKAGGBAG_FIELD_TCP_STATE: skAggBagAggregateSetUnsigned( &key, &k_it, (rwRecGetTcpState(&rwrec) & SK_TCPSTATE_ATTRIBUTE_MASK)); break; case SKAGGBAG_FIELD_APPLICATION: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetApplication(&rwrec)); break; case SKAGGBAG_FIELD_FTYPE_CLASS: skAggBagAggregateSetUnsigned( &key, &k_it, sksiteFlowtypeGetClassID(rwRecGetFlowType(&rwrec))); break; case SKAGGBAG_FIELD_FTYPE_TYPE: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetFlowType(&rwrec)); break; case SKAGGBAG_FIELD_STARTTIME: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetStartSeconds(&rwrec)); break; case SKAGGBAG_FIELD_ELAPSED: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetElapsedSeconds(&rwrec)); break; case SKAGGBAG_FIELD_ENDTIME: skAggBagAggregateSetUnsigned( &key, &k_it, rwRecGetEndSeconds(&rwrec)); break; default: break; } } while (skAggBagFieldIterNext(&k_it) == SK_ITERATOR_OK); skAggBagInitializeCounter(ab, &counter, &c_it); do { switch (skAggBagFieldIterGetType(&c_it)) { case SKAGGBAG_FIELD_RECORDS: skAggBagAggregateSetUnsigned(&counter, &c_it, 1); break; case SKAGGBAG_FIELD_SUM_BYTES: skAggBagAggregateSetUnsigned( &counter, &c_it, rwRecGetBytes(&rwrec)); break; case SKAGGBAG_FIELD_SUM_PACKETS: skAggBagAggregateSetUnsigned( &counter, &c_it, rwRecGetPkts(&rwrec)); break; case SKAGGBAG_FIELD_SUM_ELAPSED: skAggBagAggregateSetUnsigned( &counter, &c_it, rwRecGetElapsedSeconds(&rwrec)); break; default: break; } } while (skAggBagFieldIterNext(&c_it) == SK_ITERATOR_OK); err = skAggBagKeyCounterAdd(ab, &key, &counter, NULL); if (err) { skAppPrintErr("Unable to add to key: %s", skAggBagStrerror(err)); break; } } if (rv != SKSTREAM_ERR_EOF) { skStreamPrintLastErr(stream, rv, &skAppPrintErr); return -1; } return 0; }
/* * status = processInputFile(filein); * * For every line in 'filein', look up the address in Country Code * map and print out the corresponding country code. There should * be as many lines of output as there are of input. */ static int processInputFile( const char *f_name) { char final_delim[] = {'\0', '\0'}; char line[2048]; skstream_t *stream = NULL; skIPWildcardIterator_t iter; skIPWildcard_t ipwild; skipaddr_t ip; int retval = 1; int rv; int lc = 0; char cc[32]; char ipbuf[SK_NUM2DOT_STRLEN]; if (!app_opt.no_final_delimiter) { final_delim[0] = app_opt.column_separator; } /* open input */ if ((rv = skStreamCreate(&stream, SK_IO_READ, SK_CONTENT_TEXT)) || (rv = skStreamBind(stream, f_name)) || (rv = skStreamSetCommentStart(stream, "#")) || (rv = skStreamOpen(stream))) { skStreamPrintLastErr(stream, rv, &skAppPrintErr); goto END; } /* read until end of file */ while ((rv = skStreamGetLine(stream, line, sizeof(line), &lc)) != SKSTREAM_ERR_EOF) { switch (rv) { case SKSTREAM_OK: /* good, we got our line */ break; case SKSTREAM_ERR_LONG_LINE: /* bad: line was longer than sizeof(line) */ skAppPrintErr("Input line %d too long. ignored", lc); continue; default: /* unexpected error */ skStreamPrintLastErr(stream, rv, &skAppPrintErr); goto END; } /* parse the line: fill in octet_bitmap */ rv = skStringParseIPWildcard(&ipwild, line); if (rv && rv != SKUTILS_ERR_EMPTY) { /* error */ skAppPrintErr("Error on line %d: %s\n", lc, skStringParseStrerror(rv)); goto END; } #if SK_ENABLE_IPV6 if (skIPWildcardIsV6(&ipwild)) { continue; } #endif /* SK_ENABLE_IPV6 */ skIPWildcardIteratorBind(&iter, &ipwild); while (skIPWildcardIteratorNext(&iter, &ip) == SK_ITERATOR_OK) { skCountryLookupName(&ip, cc, sizeof(cc)); if (!app_opt.print_ips) { skStreamPrint(out, "%s\n", cc); } else { skipaddrString(ipbuf, &ip, ip_flags); if (app_opt.no_columns) { skStreamPrint(out, "%s%c%s%s\n", ipbuf, app_opt.column_separator, cc, final_delim); } else { skStreamPrint(out, "%15s%c%2s%s\n", ipbuf, app_opt.column_separator, cc, final_delim); } } } } retval = 0; END: skStreamDestroy(&stream); return retval; }
/* * sortRandom(); * * Don't make any assumptions about the input. Store the input * records in a large buffer, and sort those in-core records once * all records are processed or the buffer is full. If the buffer * fills up, store the sorted records into temporary files. Once * all records are read, use mergeFiles() above to merge-sort the * temporary files. * * Exits the application if an error occurs. */ static void sortRandom( void) { int temp_file_idx = -1; skstream_t *input_rwios = NULL; /* input stream */ uint8_t *record_buffer = NULL; /* Region of memory for records */ uint8_t *cur_node = NULL; /* Ptr into record_buffer */ uint8_t *next_node = NULL; /* Ptr into record_buffer */ uint32_t buffer_max_recs; /* max buffer size (in number of recs) */ uint32_t buffer_recs; /* current buffer size (# records) */ uint32_t buffer_chunk_recs; /* how to grow from current to max buf */ uint32_t num_chunks; /* how quickly to grow buffer */ uint32_t record_count = 0; /* Number of records read */ int rv; /* Determine the maximum number of records that will fit into the * buffer if it grows the maximum size */ buffer_max_recs = buffer_size / NODE_SIZE; TRACEMSG((("buffer_size = %" PRIu64 "\nnode_size = %" PRIu32 "\nbuffer_max_recs = %" PRIu32), buffer_size, NODE_SIZE, buffer_max_recs)); /* We will grow to the maximum size in chunks */ num_chunks = NUM_CHUNKS; if (num_chunks <= 0) { num_chunks = 1; } /* Attempt to allocate the initial chunk. If we fail, increment * the number of chunks---which will decrease the amount we * attempt to allocate at once---and try again. */ for (;;) { buffer_chunk_recs = buffer_max_recs / num_chunks; TRACEMSG((("num_chunks = %" PRIu32 "\nbuffer_chunk_recs = %" PRIu32), num_chunks, buffer_chunk_recs)); record_buffer = (uint8_t*)malloc(NODE_SIZE * buffer_chunk_recs); if (record_buffer) { /* malloc was successful */ break; } else if (buffer_chunk_recs < MIN_IN_CORE_RECORDS) { /* give up at this point */ skAppPrintErr("Error allocating space for %d records", MIN_IN_CORE_RECORDS); appExit(EXIT_FAILURE); } else { /* reduce the amount we allocate at once by increasing the * number of chunks and try again */ TRACEMSG(("malloc() failed")); ++num_chunks; } } buffer_recs = buffer_chunk_recs; TRACEMSG((("buffer_recs = %" PRIu32), buffer_recs)); /* open first file */ rv = appNextInput(&input_rwios); if (rv < 0) { free(record_buffer); appExit(EXIT_FAILURE); } record_count = 0; cur_node = record_buffer; while (input_rwios != NULL) { /* read record */ if ((rv = skStreamReadRecord(input_rwios, (rwRec*)cur_node)) != SKSTREAM_OK) { if (rv != SKSTREAM_ERR_EOF) { skStreamPrintLastErr(input_rwios, rv, &skAppPrintErr); } /* end of file: close current and open next */ skStreamDestroy(&input_rwios); rv = appNextInput(&input_rwios); if (rv < 0) { free(record_buffer); appExit(EXIT_FAILURE); } continue; } ++record_count; cur_node += NODE_SIZE; if (record_count == buffer_recs) { /* Filled the current buffer */ /* If buffer not at max size, see if we can grow it */ if (buffer_recs < buffer_max_recs) { uint8_t *old_buf = record_buffer; /* add a chunk of records. if we are near the max, * set the size to the max */ buffer_recs += buffer_chunk_recs; if (buffer_recs + buffer_chunk_recs > buffer_max_recs) { buffer_recs = buffer_max_recs; } TRACEMSG((("Buffer full---attempt to grow to %" PRIu32 " records, %" PRIu32 " bytes"), buffer_recs, NODE_SIZE * buffer_recs)); /* attempt to grow */ record_buffer = (uint8_t*)realloc(record_buffer, NODE_SIZE * buffer_recs); if (record_buffer) { /* Success, make certain cur_node points into the * new buffer */ cur_node = (record_buffer + (record_count * NODE_SIZE)); } else { /* Unable to grow it */ TRACEMSG(("realloc() failed")); record_buffer = old_buf; buffer_max_recs = buffer_recs = record_count; } } /* Either buffer at maximum size or attempt to grow it * failed. */ if (record_count == buffer_max_recs) { /* Sort */ skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare); /* Write to temp file */ if (skTempFileWriteBufferStream( tmpctx, &temp_file_idx, record_buffer, NODE_SIZE, record_count)) { skAppPrintSyserror( "Error writing sorted buffer to temporary file"); free(record_buffer); appExit(EXIT_FAILURE); } /* Reset record buffer to 'empty' */ record_count = 0; cur_node = record_buffer; } } } /* Sort (and maybe store) last batch of records */ if (record_count > 0) { skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare); if (temp_file_idx >= 0) { /* Write last batch to temp file */ if (skTempFileWriteBufferStream( tmpctx, &temp_file_idx, record_buffer, NODE_SIZE, record_count)) { skAppPrintSyserror( "Error writing sorted buffer to temporary file"); free(record_buffer); appExit(EXIT_FAILURE); } } } /* Generate the output */ if (record_count == 0 && temp_file_idx == -1) { /* No records were read at all; write the header to the output * file */ rv = skStreamWriteSilkHeader(out_rwios); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); } } else if (temp_file_idx == -1) { /* No temp files written, just output batch of records */ uint32_t c; TRACEMSG((("Writing %" PRIu32 " records to '%s'"), record_count, skStreamGetPathname(out_rwios))); /* get first two records from the sorted buffer */ cur_node = record_buffer; next_node = record_buffer + NODE_SIZE; for (c = 1; c < record_count; ++c, next_node += NODE_SIZE) { if (0 != rwrecCompare(cur_node, next_node)) { /* records differ. print earlier record */ rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); if (SKSTREAM_ERROR_IS_FATAL(rv)) { free(record_buffer); appExit(EXIT_FAILURE); } } cur_node = next_node; } /* else records are duplicates: ignore latter record */ } /* print remaining record */ rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); if (SKSTREAM_ERROR_IS_FATAL(rv)) { free(record_buffer); appExit(EXIT_FAILURE); } } } else { /* no longer have a need for the record buffer */ free(record_buffer); record_buffer = NULL; /* now merge all the temp files */ mergeFiles(temp_file_idx); } if (record_buffer) { free(record_buffer); } }
/* * mergeFiles(temp_file_idx) * * Merge the temporary files numbered from 0 to 'temp_file_idx' * inclusive into the output file 'out_ios', maintaining sorted * order. Exits the application if an error occurs. */ static void mergeFiles( int temp_file_idx) { char errbuf[2 * PATH_MAX]; skstream_t *fps[MAX_MERGE_FILES]; uint8_t recs[MAX_MERGE_FILES][NODE_SIZE]; uint8_t lowest_rec[NODE_SIZE]; int j; uint16_t open_count; uint16_t i; uint16_t lowest; uint16_t *top_heap; int tmp_idx_a; int tmp_idx_b; skstream_t *fp_intermediate = NULL; int tmp_idx_intermediate; skheap_t *heap; uint32_t heap_count; int opened_all_temps = 0; ssize_t rv; /* the index of the first temp file to the merge */ tmp_idx_a = 0; TRACEMSG(("Merging #%d through #%d to '%s'", tmp_idx_a, temp_file_idx, skStreamGetPathname(out_rwios))); heap = skHeapCreate2(compHeapNodes, MAX_MERGE_FILES, sizeof(uint16_t), NULL, recs); if (NULL == heap) { skAppPrintOutOfMemory("heap"); appExit(EXIT_FAILURE); } /* This loop repeats as long as we haven't read all of the temp * files generated in the qsort stage. */ do { assert(SKHEAP_ERR_EMPTY==skHeapPeekTop(heap,(skheapnode_t*)&top_heap)); /* the index of the list temp file to merge */ tmp_idx_b = temp_file_idx; /* open an intermediate temp file. The merge-sort will have * to write records here if there are not enough file handles * available to open all the existing tempoary files. */ fp_intermediate = skTempFileCreateStream(tmpctx, &tmp_idx_intermediate); if (fp_intermediate == NULL) { skAppPrintSyserror("Error creating new temporary file"); appExit(EXIT_FAILURE); } /* count number of files we open */ open_count = 0; /* Attempt to open up to MAX_MERGE_FILES, though we an open * may fail due to lack of resources (EMFILE or ENOMEM) */ for (j = tmp_idx_a; j <= tmp_idx_b; ++j) { fps[open_count] = skTempFileOpenStream(tmpctx, j); if (fps[open_count] == NULL) { if ((open_count > 0) && ((errno == EMFILE) || (errno == ENOMEM))) { /* We cannot open any more files. Rewind counter * by one to catch this file on the next merge */ tmp_idx_b = j - 1; TRACEMSG((("FILE limit hit--" "merging #%d through #%d into #%d: %s"), tmp_idx_a, tmp_idx_b, tmp_idx_intermediate, strerror(errno))); break; } else { skAppPrintSyserror(("Error opening existing" " temporary file '%s'"), skTempFileGetName(tmpctx, j)); appExit(EXIT_FAILURE); } } /* read the first record */ rv = skStreamRead(fps[open_count], recs[open_count], NODE_SIZE); if (NODE_SIZE == rv) { /* insert the file index into the heap */ skHeapInsert(heap, &open_count); ++open_count; if (open_count == MAX_MERGE_FILES) { /* We've reached the limit for this pass. Set * tmp_idx_b to the file we just opened. */ tmp_idx_b = j; TRACEMSG((("MAX_MERGE_FILES limit hit--" "merging #%d through #%d to #%d"), tmp_idx_a, tmp_idx_b, tmp_idx_intermediate)); break; } } else if (0 == rv) { TRACEMSG(("Ignoring empty temporary file '%s'", skTempFileGetName(tmpctx, j))); skStreamDestroy(&fps[open_count]); } else { if (rv > 0) { snprintf(errbuf, sizeof(errbuf), "Short read %" SK_PRIdZ "/%" PRIu32 " from '%s'", rv, NODE_SIZE, skStreamGetPathname(fps[open_count])); } else { skStreamLastErrMessage( fps[open_count], rv, errbuf, sizeof(errbuf)); } skAppPrintErr( "Error reading first record from temporary file: %s", errbuf); appExit(EXIT_FAILURE); } } /* Here, we check to see if we've opened all temp files. If * so, set a flag so we write data to final destination and * break out of the loop after we're done. */ if (tmp_idx_b == temp_file_idx) { opened_all_temps = 1; /* no longer need the intermediate temp file */ skStreamDestroy(&fp_intermediate); } else { /* we could not open all temp files, so merge all opened * temp files into the intermediate file. Add the * intermediate file to the list of files to merge */ temp_file_idx = tmp_idx_intermediate; } TRACEMSG((("Merging %" PRIu16 " temporary files"), open_count)); heap_count = skHeapGetNumberEntries(heap); assert(heap_count == open_count); /* get the index of the file with the lowest record; which is * at the top of the heap */ if (skHeapPeekTop(heap, (skheapnode_t*)&top_heap) != SKHEAP_OK) { skAppPrintErr("Unable to open and read any temporary files."); appExit(EXIT_FAILURE); } lowest = *top_heap; /* exit this do...while() once all records for all opened * files have been read */ do { /* lowest_rec is the record pointed to by the index at the * top of the heap */ memcpy(lowest_rec, recs[lowest], NODE_SIZE); /* write the record */ if (fp_intermediate) { /* write the record to intermediate tmp file */ rv = skStreamWrite(fp_intermediate, lowest_rec, NODE_SIZE); if (NODE_SIZE != rv) { skAppPrintSyserror( "Error writing record to temporary file '%s'", skTempFileGetName(tmpctx, tmp_idx_intermediate)); appExit(EXIT_FAILURE); } } else { /* we successfully opened all (remaining) temp files, * write to record to the final destination */ rv = skStreamWriteRecord(out_rwios, (rwRec*)lowest_rec); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); if (SKSTREAM_ERROR_IS_FATAL(rv)) { appExit(EXIT_FAILURE); } } } /* replace the record we just processed and loop over all * files until we get a record that is not a duplicate */ do { if ((rv = skStreamRead(fps[lowest], recs[lowest], NODE_SIZE)) != NODE_SIZE) { /* read failed. there is no more data for this * file; remove it from the heap; if the heap is * empty, exit the loop */ skHeapExtractTop(heap, NULL); --heap_count; #if TRACEMSG_LEVEL > 0 if (rv == 0) { TRACEMSG( ("Finished reading file #%u: EOF; %u files remain", (tmp_idx_a + lowest), heap_count)); } else if (rv > 0) { TRACEMSG( ("Finished reading file #%u: Short read " "%" SK_PRIdZ "/%" PRIu32 "; %u files remain", tmp_idx_a + lowest, rv, NODE_SIZE, heap_count)); } else { skStreamLastErrMessage( fps[open_count], rv, errbuf, sizeof(errbuf)); TRACEMSG( ("Finished reading file #%u: %s; %u files remain", (tmp_idx_a + lowest), errbuf, heap_count)); } #endif /* TRACEMSG_LEVEL */ if (0 == heap_count) { break; } } else if (rwrecCompare(lowest_rec, recs[lowest])) { /* read succeeded. new record is not a * duplicate and we insert it into the heap */ /* FIXME: This comparison reduces work when the * keys are the same, but it adds another * comparison when the keys are different; is this * an overall win or lose? */ skHeapReplaceTop(heap, &lowest, NULL); } else { /* read succeeded. record is a duplicate; ignore * the record and leave the heap unchanged */ continue; } /* get the record at the top of the heap and see if it * is a duplicate; if it is, ignore it. */ skHeapPeekTop(heap, (skheapnode_t*)&top_heap); lowest = *top_heap; } while (0 == rwrecCompare(lowest_rec, recs[lowest])); } while (heap_count > 0); TRACEMSG((("Finished processing #%d through #%d"), tmp_idx_a, tmp_idx_b)); /* Close all open temp files */ for (i = 0; i < open_count; ++i) { skStreamDestroy(&fps[i]); } /* Delete all temp files we opened (or attempted to open) this * time */ for (j = tmp_idx_a; j <= tmp_idx_b; ++j) { skTempFileRemove(tmpctx, j); } /* Close the intermediate temp file. */ if (fp_intermediate) { rv = skStreamClose(fp_intermediate); if (rv) { skStreamLastErrMessage( fp_intermediate, rv, errbuf, sizeof(errbuf)); skAppPrintErr("Error closing temporary file: %s", errbuf); skStreamDestroy(&fp_intermediate); appExit(EXIT_FAILURE); } skStreamDestroy(&fp_intermediate); } /* Start the next merge with the next input temp file */ tmp_idx_a = tmp_idx_b + 1; } while (!opened_all_temps); skHeapFree(heap); }
/* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup( int argc, char **argv) { SILK_FEATURES_DEFINE_STRUCT(features); unsigned int optctx_flags; int rv; /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skAppVerifyFeatures(&features, NULL); skOptionsSetUsageCallback(&appUsageLong); /* initialize globals */ memset(&ab_options, 0, sizeof(sk_aggbag_options_t)); ab_options.existing_silk_files = 1; ab_options.argc = argc; ab_options.argv = argv; optctx_flags = (SK_OPTIONS_CTX_INPUT_SILK_FLOW | SK_OPTIONS_CTX_ALLOW_STDIN | SK_OPTIONS_CTX_XARGS | SK_OPTIONS_CTX_COPY_INPUT | SK_OPTIONS_CTX_PRINT_FILENAMES); /* register the options */ if (skOptionsCtxCreate(&optctx, optctx_flags) || skOptionsCtxOptionsRegister(optctx) || skOptionsRegister(appOptions, &appOptionsHandler, NULL) || skAggBagOptionsRegister(&ab_options) || skIPv6PolicyOptionsRegister(&ipv6_policy) || sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE)) { skAppPrintErr("Unable to register options"); exit(EXIT_FAILURE); } /* register the teardown handler */ if (atexit(appTeardown) < 0) { skAppPrintErr("Unable to register appTeardown() with atexit()"); exit(EXIT_FAILURE); } /* parse options */ rv = skOptionsCtxOptionsParse(optctx, argc, argv); if (rv < 0) { skAppUsage(); /* never returns */ } /* try to load site config file; if it fails, we will not be able * to resolve flowtype and sensor from input file names, but we * should not consider it a complete failure */ sksiteConfigure(0); /* make sure the user specified at least one key field and one * counter field */ if (keys_arg == NULL || keys_arg[0] == '\0') { skAppPrintErr("The --%s switch is required", appOptions[OPT_KEYS].name); skAppUsage(); /* never returns */ } if (counters_arg == NULL || counters_arg[0] == '\0') { skAppPrintErr("The --%s switch is required", appOptions[OPT_COUNTERS].name); skAppUsage(); /* never returns */ } /* set up the key_name_map and counter_name_map */ if (createStringmaps()) { exit(EXIT_FAILURE); } /* create the aggregate bag */ if (skAggBagCreate(&ab)) { exit(EXIT_FAILURE); } skAggBagOptionsBind(ab, &ab_options); /* parse the --keys and --counters switches */ if (parseFields(key_name_map, keys_arg, OPT_KEYS)) { exit(EXIT_FAILURE); } if (parseFields(counter_name_map, counters_arg, OPT_COUNTERS)) { exit(EXIT_FAILURE); } /* create output stream to stdout if no --output-path was given */ if (NULL == output) { if ((rv = skStreamCreate(&output, SK_IO_WRITE, SK_CONTENT_SILK)) || (rv = skStreamBind(output, "-"))) { skStreamPrintLastErr(output, rv, &skAppPrintErr); skStreamDestroy(&output); exit(EXIT_FAILURE); } } /* make certain stdout is not being used for multiple outputs */ if (skStreamIsStdout(output) && skOptionsCtxCopyStreamIsStdout(optctx)) { skAppPrintErr("May not use stdout for multiple output streams"); exit(EXIT_FAILURE); } /* open the output stream but do not write anything yet */ rv = skStreamOpen(output); if (rv) { skStreamPrintLastErr(output, rv, &skAppPrintErr); skStreamDestroy(&output); exit(EXIT_FAILURE); } /* open the --copy-input stream */ if (skOptionsCtxOpenStreams(optctx, &skAppPrintErr)) { exit(EXIT_FAILURE); } return; /* OK */ }
/* * buildIPSetRanges(stream); * * Read IP addresses from the stream named by 'stream' and use them * to build the global ipset. Allow the input to support ranges of * IPs. Return 0 on success or -1 on failure. */ static int buildIPSetRanges( skstream_t *stream) { #if SK_ENABLE_IPV6 int saw_integer = 0; #endif int lc = 0; char line_buf[512]; char *sep; skipaddr_t ip; skipaddr_t ip_min; skipaddr_t ip_max; uint32_t prefix; const int delim_is_space = isspace((int)delimiter); int rv; /* read until end of file */ while ((rv = skStreamGetLine(stream, line_buf, sizeof(line_buf), &lc)) != SKSTREAM_ERR_EOF) { switch (rv) { case SKSTREAM_OK: /* good, we got our line */ break; case SKSTREAM_ERR_LONG_LINE: /* bad: line was longer than sizeof(line_buf) */ skAppPrintErr("Input line %d too long. ignored", lc); continue; default: /* unexpected error */ skStreamPrintLastErr(stream, rv, &skAppPrintErr); goto END; } /* support whitespace separators */ if (!delim_is_space) { sep = strchr(line_buf, delimiter); } else { /* ignore leading whitespace */ sep = line_buf; while (isspace((int)*sep)) { ++sep; } sep = strchr(sep, delimiter); if (sep) { /* allow a lone IP to have trailing whitespace */ char *cp = sep; while (isspace((int)*cp)) { ++cp; } if (*cp == '\0') { sep = NULL; } } } if (sep == NULL) { /* parse as IP with possible CIDR designation */ rv = skStringParseCIDR(&ip, &prefix, line_buf); if (rv != 0) { skAppPrintErr("Invalid IP on line %d: %s", lc, skStringParseStrerror(rv)); goto END; } #if SK_ENABLE_IPV6 /* do not allow integers mixed with IPv6 addresses */ if (saw_integer) { if (skipaddrIsV6(&ip)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } } else if (SETBUILD_BUF_IS_INT(line_buf)) { saw_integer = 1; if (skIPSetIsV6(ipset)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } } #endif /* SK_ENABLE_IPV6 */ rv = skIPSetInsertAddress(ipset, &ip, prefix); if (rv) { skAppPrintErr("Error adding IP on line %d to IPset: %s", lc, skIPSetStrerror(rv)); goto END; } continue; } /* parse two IP addresses */ *sep = '\0'; ++sep; rv = skStringParseIP(&ip_min, line_buf); if (rv != 0) { skAppPrintErr("Invalid minimum IP on line %d: %s", lc, skStringParseStrerror(rv)); goto END; } rv = skStringParseIP(&ip_max, sep); if (rv != 0) { skAppPrintErr("Invalid maximum IP on line %d: %s", lc, skStringParseStrerror(rv)); goto END; } if (skipaddrCompare(&ip_min, &ip_max) > 0) { skAppPrintErr("Invalid IP range on line %d: min > max", lc); rv = -1; goto END; } #if SK_ENABLE_IPV6 /* do not allow integers mixed with IPv6 addresses */ if (saw_integer) { if (skipaddrIsV6(&ip_min) || skipaddrIsV6(&ip_max)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } } else if (SETBUILD_BUF_IS_INT(line_buf) || SETBUILD_BUF_IS_INT(sep)) { saw_integer = 1; if (skIPSetIsV6(ipset)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } } #endif /* SK_ENABLE_IPV6 */ rv = skIPSetInsertRange(ipset, &ip_min, &ip_max); if (rv) { skAppPrintErr("Error adding IP range on line %d to IPset: %s", lc, skIPSetStrerror(rv)); goto END; } } /* success */ rv = 0; END: if (rv != 0) { return -1; } return 0; }
/* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup( int argc, char **argv) { SILK_FEATURES_DEFINE_STRUCT(features); const char *in_fname; const char *out_fname; int arg_index; int rv; /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skAppVerifyFeatures(&features, NULL); skOptionsSetUsageCallback(&appUsageLong); /* initialize globals */ memset(&set_options, 0, sizeof(skipset_options_t)); set_options.argc = argc; set_options.argv = argv; /* register the options */ if (skOptionsRegister(appOptions, &appOptionsHandler, NULL) || skIPSetOptionsRegister(&set_options)) { skAppPrintErr("Unable to register options"); exit(EXIT_FAILURE); } /* register the teardown handler */ if (atexit(appTeardown) < 0) { skAppPrintErr("Unable to register appTeardown() with atexit()"); appTeardown(); exit(EXIT_FAILURE); } /* parse the options */ arg_index = skOptionsParse(argc, argv); if (arg_index < 0) { skAppUsage(); /* never returns */ } /* default is to read from stdin and write to stdout */ in_fname = "-"; out_fname = "-"; /* process files named on the command line */ switch (argc - arg_index) { case 2: in_fname = argv[arg_index++]; out_fname = argv[arg_index++]; break; case 1: in_fname = argv[arg_index++]; break; case 0: /* Do not allow reading from a tty when no input */ if (FILEIsATty(stdin)) { skAppPrintErr("Must specify '-' as the input to read" " from a terminal"); exit(EXIT_FAILURE); } break; default: skAppPrintErr("Too many arguments;" " a maximum of two files may be specified"); skAppUsage(); } /* we should have processed all arguments */ assert(arg_index == argc); /* create the IPset */ if (skIPSetCreate(&ipset, 0)) { EXIT_NO_MEMORY; } skIPSetOptionsBind(ipset, &set_options); /* create input */ if ((rv = skStreamCreate(&in_stream, SK_IO_READ, SK_CONTENT_TEXT)) || (rv = skStreamBind(in_stream, in_fname)) || (rv = skStreamSetCommentStart(in_stream, "#"))) { skStreamPrintLastErr(in_stream, rv, &skAppPrintErr); exit(EXIT_FAILURE); } /* create output */ if ((rv = skStreamCreate(&out_stream, SK_IO_WRITE, SK_CONTENT_SILK)) || (rv = skStreamBind(out_stream, out_fname))) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); exit(EXIT_FAILURE); } /* open the streams */ rv = skStreamOpen(in_stream); if (rv) { skStreamPrintLastErr(in_stream, rv, &skAppPrintErr); exit(EXIT_FAILURE); } rv = skStreamOpen(out_stream); if (rv) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); exit(EXIT_FAILURE); } }
/* * status = appOptionsHandler(cData, opt_index, opt_arg); * * This function is passed to skOptionsRegister(); it will be called * by skOptionsParse() for each user-specified switch that the * application has registered; it should handle the switch as * required---typically by setting global variables---and return 1 * if the switch processing failed or 0 if it succeeded. Returning * a non-zero from from the handler causes skOptionsParse() to return * a negative value. * * The clientData in 'cData' is typically ignored; 'opt_index' is * the index number that was specified as the last value for each * struct option in appOptions[]; 'opt_arg' is the user's argument * to the switch for options that have a REQUIRED_ARG or an * OPTIONAL_ARG. */ static int appOptionsHandler( clientData UNUSED(cData), int opt_index, char *opt_arg) { size_t sz; int rv; switch ((appOptionsEnum)opt_index) { case OPT_SILK_OUTPUT: if (silk_output) { skAppPrintErr("Invalid %s: Switch used multiple times", appOptions[opt_index].name); return 1; } if ((rv =skStreamCreate(&silk_output,SK_IO_WRITE,SK_CONTENT_SILK_FLOW)) || (rv = skStreamBind(silk_output, opt_arg))) { skStreamPrintLastErr(silk_output, rv, &skAppPrintErr); exit(EXIT_FAILURE); } break; case OPT_PRINT_STATISTICS: print_statistics = 1; break; case OPT_LOG_DESTINATION: if ('\0' != log_destination[0]) { skAppPrintErr("Invalid %s: Switch used multiple times", appOptions[opt_index].name); } if ('\0' == opt_arg[0]) { skAppPrintErr("Invalid %s: Path name is required", appOptions[opt_index].name); return 1; } if (0 == strcmp("stdout", opt_arg) || 0 == strcmp("stderr", opt_arg) || 0 == strcmp("none", opt_arg)) { strncpy(log_destination, opt_arg, sizeof(log_destination)); break; } if ('/' == opt_arg[0]) { if (strlen(opt_arg) >= sizeof(log_destination)) { skAppPrintErr("Invalid %s: Name is too long", appOptions[opt_index].name); return 1; } strncpy(log_destination, opt_arg, sizeof(log_destination)); break; } if (NULL == getcwd(log_destination, sizeof(log_destination))) { skAppPrintSyserror("Unable to get current directory"); return 1; } sz = strlen(log_destination); if (sz + strlen(opt_arg) + 1 >= sizeof(log_destination)) { skAppPrintErr("Invalid %s: Name is too long", appOptions[opt_index].name); return 1; } snprintf(log_destination + sz, sizeof(log_destination) - sz, "/%s", opt_arg); break; case OPT_LOG_FLAGS: if (log_flags) { skAppPrintErr("Invaild %s: Switch used multiple times", appOptions[opt_index].name); return 1; } log_flags = opt_arg; break; } return 0; /* OK */ }
/* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup( int argc, char **argv) { SILK_FEATURES_DEFINE_STRUCT(features); int rv; /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skAppVerifyFeatures(&features, NULL); skOptionsSetUsageCallback(&appUsageLong); /* initialize globals */ #error "Initialize any global variables here" /* for example: set global output to NULL */ out_stream = NULL; /* register the options */ if (skOptionsRegister(appOptions, &appOptionsHandler, NULL) || sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE)) { skAppPrintErr("Unable to register options"); exit(EXIT_FAILURE); } #error "Do any other module setup here" /* register the teardown handler */ if (atexit(appTeardown) < 0) { skAppPrintErr("Unable to register appTeardown() with atexit()"); appTeardown(); exit(EXIT_FAILURE); } /* parse the options */ arg_index = skOptionsParse(argc, argv); if (arg_index < 0) { /* options parsing should print error */ skAppUsage(); /* never returns */ } /* try to load site config file; if it fails, we will not be able * to resolve flowtype and sensor from input file names */ sksiteConfigure(0); #error "Do any options validation here" #error "If you expect filenames on command line keep this:" /* arg_index is looking at first file name to process */ if (arg_index == argc) { if (FILEIsATty(stdin)) { skAppPrintErr("No input files on command line and" " stdin is connected to a terminal"); skAppUsage(); /* never returns */ } } #error "If you do NOT expect filenames on command line, keep this:" /* check for extraneous arguments */ if (arg_index != argc) { skAppPrintErr("Too many arguments or unrecognized switch '%s'", argv[arg_index]); skAppUsage(); /* never returns */ } #error "Once all options are set, open input and output" /* for example, open a SiLK flow file as an output file */ rv = skStreamOpenSilkFlow(&out_stream, output_path, SK_IO_WRITE); if (rv) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); skAppPrintErr("Unable to open output file. Exiting"); skStreamDestroy(&out_stream); exit(EXIT_FAILURE); } return; /* OK */ }
/* * buildIPSetWildcards(stream); * * Read IP addresses from the stream named by 'stream' and use them * to build the global ipset. Allow the input to contain * IPWildcards. Return 0 on success or -1 on failure. */ static int buildIPSetWildcards( skstream_t *stream) { #if SK_ENABLE_IPV6 int saw_integer = 0; #endif int lc = 0; char line_buf[512]; skIPWildcard_t ipwild; skipaddr_t ip; uint32_t prefix; char *cp; int rv; /* read until end of file */ while ((rv = skStreamGetLine(stream, line_buf, sizeof(line_buf), &lc)) != SKSTREAM_ERR_EOF) { switch (rv) { case SKSTREAM_OK: /* good, we got our line */ break; case SKSTREAM_ERR_LONG_LINE: /* bad: line was longer than sizeof(line_buf) */ skAppPrintErr("Input line %d too long. ignored", lc); continue; default: /* unexpected error */ skStreamPrintLastErr(stream, rv, &skAppPrintErr); goto END; } /* first, attempt to parse as a CIDR block */ rv = skStringParseCIDR(&ip, &prefix, line_buf); if (rv == 0) { #if SK_ENABLE_IPV6 /* do not allow integers mixed with IPv6 addresses */ if (saw_integer) { if (skipaddrIsV6(&ip)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } } else if (SETBUILD_BUF_IS_INT(line_buf)) { saw_integer = 1; if (skIPSetIsV6(ipset)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } } #endif /* SK_ENABLE_IPV6 */ rv = skIPSetInsertAddress(ipset, &ip, prefix); if (rv) { skAppPrintErr("Error adding IP on line %d to IPset: %s", lc, skIPSetStrerror(rv)); goto END; } continue; } /* else parse the line as an IPWildcard */ rv = skStringParseIPWildcard(&ipwild, line_buf); if (rv != 0) { /* failed to parse an IPWildcard. See if the user has * entered two IP addresses, and if so, suggest they use * the --ip-ranges switch. */ int rv2; rv2 = skStringParseIP(&ip, line_buf); if (rv2 > 0) { /* parsed an IP and there is extra text after the IP * address; check to see if it is another IP addr */ #if SK_ENABLE_IPV6 if (skipaddrIsV6(&ip) && ((cp = strchr(line_buf + rv2, ':')) != NULL)) { while (isxdigit((int) *(cp - 1))) { --cp; } if (skStringParseIP(&ip, cp) == 0) { skAppPrintErr(("Invalid IP on line %d: " SUGGEST_IP_RANGES), lc); goto END; } } #endif if (!skipaddrIsV6(&ip) && ((cp = strchr(line_buf + rv2, '.')) != NULL)) { while (isxdigit((int) *(cp - 1))) { --cp; } if (skStringParseIP(&ip, cp) == 0) { skAppPrintErr(("Invalid IP on line %d: " SUGGEST_IP_RANGES), lc); goto END; } } } /* report initial error */ skAppPrintErr("Invalid IP Wildcard on line %d: %s", lc, skStringParseStrerror(rv)); goto END; } #if SK_ENABLE_IPV6 /* do not allow integers mixed with IPv6 addresses */ if (saw_integer && skIPWildcardIsV6(&ipwild)) { skAppPrintErr("Error on line %d: %s", lc, SETBUILD_ERR_MIX_INT_V6); rv = -1; goto END; } #endif /* SK_ENABLE_IPV6 */ rv = skIPSetInsertIPWildcard(ipset, &ipwild); if (rv) { skAppPrintErr("Error adding IP Wildcard on line %d to IPset: %s", lc, skIPSetStrerror(rv)); goto END; } } /* success */ rv = 0; END: if (rv != 0) { return -1; } return 0; }
/* * status = cutFile(stream); * * Read SiLK flow records from the file at 'stream' and maybe print * them according the values in 'skip_recs' and 'num_recs'. * * Return -1 on error. Return 1 if all requested records have been * printed and processing should stop. Return 0 if processing * should continue. */ static int cutFile( skstream_t *stream) { static int copy_input_only = 0; rwRec rwrec; int rv = SKSTREAM_OK; size_t num_skipped; int ret_val = 0; /* handle case where all requested records have been printed, but * we need to write all records to the --copy-input stream. */ if (copy_input_only) { while ((rv = skStreamSkipRecords(stream, CUT_SKIP_COUNT, NULL)) == SKSTREAM_OK) ; /* empty */ if (SKSTREAM_ERR_EOF != rv) { ret_val = -1; } goto END; } /* skip any leading records */ if (skip_recs) { rv = skStreamSkipRecords(stream, skip_recs, &num_skipped); switch (rv) { case SKSTREAM_OK: skip_recs -= num_skipped; break; case SKSTREAM_ERR_EOF: skip_recs -= num_skipped; goto END; default: ret_val = -1; goto END; } } if (0 == num_recs) { /* print all records */ while ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK) { rwAsciiPrintRec(ascii_str, &rwrec); } if (SKSTREAM_ERR_EOF != rv) { ret_val = -1; } } else { while (num_recs && ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK)) { rwAsciiPrintRec(ascii_str, &rwrec); --num_recs; } switch (rv) { case SKSTREAM_OK: case SKSTREAM_ERR_EOF: break; default: ret_val = -1; goto END; } if (0 == num_recs) { if (0 == skOptionsCtxCopyStreamIsActive(optctx)) { /* we're done */ ret_val = 1; } else { /* send all remaining records to copy-input */ copy_input_only = 1; while ((rv = skStreamSkipRecords(stream, CUT_SKIP_COUNT, NULL)) == SKSTREAM_OK) ; /* empty */ if (SKSTREAM_ERR_EOF != rv) { ret_val = -1; } } } } END: if (-1 == ret_val) { skStreamPrintLastErr(stream, rv, &skAppPrintErr); } return ret_val; }