int main(int argc, char **argv)
{
skstream_t *in_stream;
int in_rv = SKSTREAM_OK;
int rv = SKSTREAM_OK;
appSetup(argc, argv); /* never returns on error */
#error "Loop over files on command line or read from stdin."
#error "Process each file, preferably in a separate function."
/* For each input, process each record */
while (NULL != (in_stream = appNextInput(argc, argv))) {
while ((in_rv = skStreamReadRecord(in_stream, &rwrec))==SKSTREAM_OK) {
/* process record */
rv = skStreamWriteRecord(out_stream, &rwrec);
if (SKSTREAM_OK != rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
skStreamDestroy(&in_stream);
goto END;
}
}
if (SKSTREAM_ERR_EOF != in_rv) {
skStreamPrintLastErr(in_stream, in_rv, &skAppPrintErr);
}
skStreamDestroy(&in_stream);
}
rv = skStreamClose(out_stream);
if (SKSTREAM_OK != rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
}
END:
return ((SKSTREAM_OK == rv) ? EXIT_SUCCESS : EXIT_FAILURE);
}
/*
* sortRandom();
*
* Don't make any assumptions about the input. Store the input
* records in a large buffer, and sort those in-core records once
* all records are processed or the buffer is full. If the buffer
* fills up, store the sorted records into temporary files. Once
* all records are read, use mergeFiles() above to merge-sort the
* temporary files.
*
* Exits the application if an error occurs.
*/
static void
sortRandom(
void)
{
int temp_file_idx = -1;
skstream_t *input_rwios = NULL; /* input stream */
uint8_t *record_buffer = NULL; /* Region of memory for records */
uint8_t *cur_node = NULL; /* Ptr into record_buffer */
uint8_t *next_node = NULL; /* Ptr into record_buffer */
uint32_t buffer_max_recs; /* max buffer size (in number of recs) */
uint32_t buffer_recs; /* current buffer size (# records) */
uint32_t buffer_chunk_recs; /* how to grow from current to max buf */
uint32_t num_chunks; /* how quickly to grow buffer */
uint32_t record_count = 0; /* Number of records read */
int rv;
/* Determine the maximum number of records that will fit into the
* buffer if it grows the maximum size */
buffer_max_recs = buffer_size / NODE_SIZE;
TRACEMSG((("buffer_size = %" PRIu64
"\nnode_size = %" PRIu32
"\nbuffer_max_recs = %" PRIu32),
buffer_size, NODE_SIZE, buffer_max_recs));
/* We will grow to the maximum size in chunks */
num_chunks = NUM_CHUNKS;
if (num_chunks <= 0) {
num_chunks = 1;
}
/* Attempt to allocate the initial chunk. If we fail, increment
* the number of chunks---which will decrease the amount we
* attempt to allocate at once---and try again. */
for (;;) {
buffer_chunk_recs = buffer_max_recs / num_chunks;
TRACEMSG((("num_chunks = %" PRIu32
"\nbuffer_chunk_recs = %" PRIu32),
num_chunks, buffer_chunk_recs));
record_buffer = (uint8_t*)malloc(NODE_SIZE * buffer_chunk_recs);
if (record_buffer) {
/* malloc was successful */
break;
} else if (buffer_chunk_recs < MIN_IN_CORE_RECORDS) {
/* give up at this point */
skAppPrintErr("Error allocating space for %d records",
MIN_IN_CORE_RECORDS);
appExit(EXIT_FAILURE);
} else {
/* reduce the amount we allocate at once by increasing the
* number of chunks and try again */
TRACEMSG(("malloc() failed"));
++num_chunks;
}
}
buffer_recs = buffer_chunk_recs;
TRACEMSG((("buffer_recs = %" PRIu32), buffer_recs));
/* open first file */
rv = appNextInput(&input_rwios);
if (rv < 0) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
record_count = 0;
cur_node = record_buffer;
while (input_rwios != NULL) {
/* read record */
if ((rv = skStreamReadRecord(input_rwios, (rwRec*)cur_node))
!= SKSTREAM_OK)
{
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(input_rwios, rv, &skAppPrintErr);
}
/* end of file: close current and open next */
skStreamDestroy(&input_rwios);
rv = appNextInput(&input_rwios);
if (rv < 0) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
continue;
}
++record_count;
cur_node += NODE_SIZE;
if (record_count == buffer_recs) {
/* Filled the current buffer */
/* If buffer not at max size, see if we can grow it */
if (buffer_recs < buffer_max_recs) {
uint8_t *old_buf = record_buffer;
/* add a chunk of records. if we are near the max,
* set the size to the max */
buffer_recs += buffer_chunk_recs;
if (buffer_recs + buffer_chunk_recs > buffer_max_recs) {
buffer_recs = buffer_max_recs;
}
TRACEMSG((("Buffer full---attempt to grow to %" PRIu32
" records, %" PRIu32 " bytes"),
buffer_recs, NODE_SIZE * buffer_recs));
/* attempt to grow */
record_buffer = (uint8_t*)realloc(record_buffer,
NODE_SIZE * buffer_recs);
if (record_buffer) {
/* Success, make certain cur_node points into the
* new buffer */
cur_node = (record_buffer + (record_count * NODE_SIZE));
} else {
/* Unable to grow it */
TRACEMSG(("realloc() failed"));
record_buffer = old_buf;
buffer_max_recs = buffer_recs = record_count;
}
}
/* Either buffer at maximum size or attempt to grow it
* failed. */
if (record_count == buffer_max_recs) {
/* Sort */
skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare);
/* Write to temp file */
if (skTempFileWriteBufferStream(
tmpctx, &temp_file_idx,
record_buffer, NODE_SIZE, record_count))
{
skAppPrintSyserror(
"Error writing sorted buffer to temporary file");
free(record_buffer);
appExit(EXIT_FAILURE);
}
/* Reset record buffer to 'empty' */
record_count = 0;
cur_node = record_buffer;
}
}
}
/* Sort (and maybe store) last batch of records */
if (record_count > 0) {
skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare);
if (temp_file_idx >= 0) {
/* Write last batch to temp file */
if (skTempFileWriteBufferStream(
tmpctx, &temp_file_idx,
record_buffer, NODE_SIZE, record_count))
{
skAppPrintSyserror(
"Error writing sorted buffer to temporary file");
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
}
/* Generate the output */
if (record_count == 0 && temp_file_idx == -1) {
/* No records were read at all; write the header to the output
* file */
rv = skStreamWriteSilkHeader(out_rwios);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
}
} else if (temp_file_idx == -1) {
/* No temp files written, just output batch of records */
uint32_t c;
TRACEMSG((("Writing %" PRIu32 " records to '%s'"),
record_count, skStreamGetPathname(out_rwios)));
/* get first two records from the sorted buffer */
cur_node = record_buffer;
next_node = record_buffer + NODE_SIZE;
for (c = 1; c < record_count; ++c, next_node += NODE_SIZE) {
if (0 != rwrecCompare(cur_node, next_node)) {
/* records differ. print earlier record */
rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
cur_node = next_node;
}
/* else records are duplicates: ignore latter record */
}
/* print remaining record */
rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
} else {
/* no longer have a need for the record buffer */
free(record_buffer);
record_buffer = NULL;
/* now merge all the temp files */
mergeFiles(temp_file_idx);
}
if (record_buffer) {
free(record_buffer);
}
}
int main(int argc, char **argv)
{
char errbuf[2 * PATH_MAX];
const char *filename = NULL;
skstream_t *stream = NULL;
skIPWildcard_t ipwild;
skipset_t *input_set = NULL;
skipset_t *wild_set = NULL;
char buf[64];
int found_match = 0; /* application return value */
int rv;
appSetup(argc, argv); /* never returns on error */
/* Build an IP wildcard from the pattern argument */
rv = skStringParseIPWildcard(&ipwild, pattern);
if (rv) {
skAppPrintErr("Invalid IP '%s': %s",
pattern, skStringParseStrerror(rv));
skAppUsage();
}
if (count && !quiet) {
/* Create an IPset containing the IPwildcard */
if ((rv = skIPSetCreate(&wild_set, skIPWildcardIsV6(&ipwild)))
|| (rv = skIPSetInsertIPWildcard(wild_set, &ipwild))
|| (rv = skIPSetClean(wild_set)))
{
skAppPrintErr("Unable to create temporary IPset: %s",
skIPSetStrerror(rv));
return EXIT_FAILURE;
}
}
/* Iterate over the set files */
while ((filename = appNextInput(argc, argv)) != NULL) {
/* Load the input set */
if ((rv = skStreamCreate(&stream, SK_IO_READ, SK_CONTENT_SILK))
|| (rv = skStreamBind(stream, filename))
|| (rv = skStreamOpen(stream)))
{
skStreamLastErrMessage(stream, rv, errbuf, sizeof(errbuf));
skAppPrintErr("Unable to read IPset from '%s': %s",
filename, errbuf);
skStreamDestroy(&stream);
continue;
}
rv = skIPSetRead(&input_set, stream);
if (rv) {
if (SKIPSET_ERR_FILEIO == rv) {
skStreamLastErrMessage(stream,
skStreamGetLastReturnValue(stream),
errbuf, sizeof(errbuf));
} else {
strncpy(errbuf, skIPSetStrerror(rv), sizeof(errbuf));
}
skAppPrintErr("Unable to read IPset from '%s': %s",
filename, errbuf);
skStreamDestroy(&stream);
continue;
}
skStreamDestroy(&stream);
if (quiet || !count) {
/* Only need to check for a match */
if (skIPSetCheckIPWildcard(input_set, &ipwild)) {
found_match = 1;
if (quiet) {
goto done;
}
printf("%s\n", filename);
}
} else {
/* Need a count of IPs, so intersect */
rv = skIPSetIntersect(input_set, wild_set);
if (rv) {
skAppPrintErr("Unable to intersect IPsets: %s",
skIPSetStrerror(rv));
skIPSetDestroy(&input_set);
skIPSetDestroy(&wild_set);
return EXIT_FAILURE;
}
printf("%s:%s\n",
filename,
skIPSetCountIPsString(input_set, buf, sizeof(buf)));
if ('0' != buf[0]) {
found_match = 1;
}
}
skIPSetDestroy(&input_set);
}
done:
/* done */
skIPSetDestroy(&input_set);
skIPSetDestroy(&wild_set);
return ((found_match) ? 0 : 1);
}
