int main(int argc, char **argv) { skstream_t *in_stream; int in_rv = SKSTREAM_OK; int rv = SKSTREAM_OK; appSetup(argc, argv); /* never returns on error */ #error "Loop over files on command line or read from stdin." #error "Process each file, preferably in a separate function." /* For each input, process each record */ while (NULL != (in_stream = appNextInput(argc, argv))) { while ((in_rv = skStreamReadRecord(in_stream, &rwrec))==SKSTREAM_OK) { /* process record */ rv = skStreamWriteRecord(out_stream, &rwrec); if (SKSTREAM_OK != rv) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); skStreamDestroy(&in_stream); goto END; } } if (SKSTREAM_ERR_EOF != in_rv) { skStreamPrintLastErr(in_stream, in_rv, &skAppPrintErr); } skStreamDestroy(&in_stream); } rv = skStreamClose(out_stream); if (SKSTREAM_OK != rv) { skStreamPrintLastErr(out_stream, rv, &skAppPrintErr); } END: return ((SKSTREAM_OK == rv) ? EXIT_SUCCESS : EXIT_FAILURE); }
/* * sortRandom(); * * Don't make any assumptions about the input. Store the input * records in a large buffer, and sort those in-core records once * all records are processed or the buffer is full. If the buffer * fills up, store the sorted records into temporary files. Once * all records are read, use mergeFiles() above to merge-sort the * temporary files. * * Exits the application if an error occurs. */ static void sortRandom( void) { int temp_file_idx = -1; skstream_t *input_rwios = NULL; /* input stream */ uint8_t *record_buffer = NULL; /* Region of memory for records */ uint8_t *cur_node = NULL; /* Ptr into record_buffer */ uint8_t *next_node = NULL; /* Ptr into record_buffer */ uint32_t buffer_max_recs; /* max buffer size (in number of recs) */ uint32_t buffer_recs; /* current buffer size (# records) */ uint32_t buffer_chunk_recs; /* how to grow from current to max buf */ uint32_t num_chunks; /* how quickly to grow buffer */ uint32_t record_count = 0; /* Number of records read */ int rv; /* Determine the maximum number of records that will fit into the * buffer if it grows the maximum size */ buffer_max_recs = buffer_size / NODE_SIZE; TRACEMSG((("buffer_size = %" PRIu64 "\nnode_size = %" PRIu32 "\nbuffer_max_recs = %" PRIu32), buffer_size, NODE_SIZE, buffer_max_recs)); /* We will grow to the maximum size in chunks */ num_chunks = NUM_CHUNKS; if (num_chunks <= 0) { num_chunks = 1; } /* Attempt to allocate the initial chunk. If we fail, increment * the number of chunks---which will decrease the amount we * attempt to allocate at once---and try again. */ for (;;) { buffer_chunk_recs = buffer_max_recs / num_chunks; TRACEMSG((("num_chunks = %" PRIu32 "\nbuffer_chunk_recs = %" PRIu32), num_chunks, buffer_chunk_recs)); record_buffer = (uint8_t*)malloc(NODE_SIZE * buffer_chunk_recs); if (record_buffer) { /* malloc was successful */ break; } else if (buffer_chunk_recs < MIN_IN_CORE_RECORDS) { /* give up at this point */ skAppPrintErr("Error allocating space for %d records", MIN_IN_CORE_RECORDS); appExit(EXIT_FAILURE); } else { /* reduce the amount we allocate at once by increasing the * number of chunks and try again */ TRACEMSG(("malloc() failed")); ++num_chunks; } } buffer_recs = buffer_chunk_recs; TRACEMSG((("buffer_recs = %" PRIu32), buffer_recs)); /* open first file */ rv = appNextInput(&input_rwios); if (rv < 0) { free(record_buffer); appExit(EXIT_FAILURE); } record_count = 0; cur_node = record_buffer; while (input_rwios != NULL) { /* read record */ if ((rv = skStreamReadRecord(input_rwios, (rwRec*)cur_node)) != SKSTREAM_OK) { if (rv != SKSTREAM_ERR_EOF) { skStreamPrintLastErr(input_rwios, rv, &skAppPrintErr); } /* end of file: close current and open next */ skStreamDestroy(&input_rwios); rv = appNextInput(&input_rwios); if (rv < 0) { free(record_buffer); appExit(EXIT_FAILURE); } continue; } ++record_count; cur_node += NODE_SIZE; if (record_count == buffer_recs) { /* Filled the current buffer */ /* If buffer not at max size, see if we can grow it */ if (buffer_recs < buffer_max_recs) { uint8_t *old_buf = record_buffer; /* add a chunk of records. if we are near the max, * set the size to the max */ buffer_recs += buffer_chunk_recs; if (buffer_recs + buffer_chunk_recs > buffer_max_recs) { buffer_recs = buffer_max_recs; } TRACEMSG((("Buffer full---attempt to grow to %" PRIu32 " records, %" PRIu32 " bytes"), buffer_recs, NODE_SIZE * buffer_recs)); /* attempt to grow */ record_buffer = (uint8_t*)realloc(record_buffer, NODE_SIZE * buffer_recs); if (record_buffer) { /* Success, make certain cur_node points into the * new buffer */ cur_node = (record_buffer + (record_count * NODE_SIZE)); } else { /* Unable to grow it */ TRACEMSG(("realloc() failed")); record_buffer = old_buf; buffer_max_recs = buffer_recs = record_count; } } /* Either buffer at maximum size or attempt to grow it * failed. */ if (record_count == buffer_max_recs) { /* Sort */ skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare); /* Write to temp file */ if (skTempFileWriteBufferStream( tmpctx, &temp_file_idx, record_buffer, NODE_SIZE, record_count)) { skAppPrintSyserror( "Error writing sorted buffer to temporary file"); free(record_buffer); appExit(EXIT_FAILURE); } /* Reset record buffer to 'empty' */ record_count = 0; cur_node = record_buffer; } } } /* Sort (and maybe store) last batch of records */ if (record_count > 0) { skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare); if (temp_file_idx >= 0) { /* Write last batch to temp file */ if (skTempFileWriteBufferStream( tmpctx, &temp_file_idx, record_buffer, NODE_SIZE, record_count)) { skAppPrintSyserror( "Error writing sorted buffer to temporary file"); free(record_buffer); appExit(EXIT_FAILURE); } } } /* Generate the output */ if (record_count == 0 && temp_file_idx == -1) { /* No records were read at all; write the header to the output * file */ rv = skStreamWriteSilkHeader(out_rwios); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); } } else if (temp_file_idx == -1) { /* No temp files written, just output batch of records */ uint32_t c; TRACEMSG((("Writing %" PRIu32 " records to '%s'"), record_count, skStreamGetPathname(out_rwios))); /* get first two records from the sorted buffer */ cur_node = record_buffer; next_node = record_buffer + NODE_SIZE; for (c = 1; c < record_count; ++c, next_node += NODE_SIZE) { if (0 != rwrecCompare(cur_node, next_node)) { /* records differ. print earlier record */ rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); if (SKSTREAM_ERROR_IS_FATAL(rv)) { free(record_buffer); appExit(EXIT_FAILURE); } } cur_node = next_node; } /* else records are duplicates: ignore latter record */ } /* print remaining record */ rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node); if (0 != rv) { skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr); if (SKSTREAM_ERROR_IS_FATAL(rv)) { free(record_buffer); appExit(EXIT_FAILURE); } } } else { /* no longer have a need for the record buffer */ free(record_buffer); record_buffer = NULL; /* now merge all the temp files */ mergeFiles(temp_file_idx); } if (record_buffer) { free(record_buffer); } }
int main(int argc, char **argv) { char errbuf[2 * PATH_MAX]; const char *filename = NULL; skstream_t *stream = NULL; skIPWildcard_t ipwild; skipset_t *input_set = NULL; skipset_t *wild_set = NULL; char buf[64]; int found_match = 0; /* application return value */ int rv; appSetup(argc, argv); /* never returns on error */ /* Build an IP wildcard from the pattern argument */ rv = skStringParseIPWildcard(&ipwild, pattern); if (rv) { skAppPrintErr("Invalid IP '%s': %s", pattern, skStringParseStrerror(rv)); skAppUsage(); } if (count && !quiet) { /* Create an IPset containing the IPwildcard */ if ((rv = skIPSetCreate(&wild_set, skIPWildcardIsV6(&ipwild))) || (rv = skIPSetInsertIPWildcard(wild_set, &ipwild)) || (rv = skIPSetClean(wild_set))) { skAppPrintErr("Unable to create temporary IPset: %s", skIPSetStrerror(rv)); return EXIT_FAILURE; } } /* Iterate over the set files */ while ((filename = appNextInput(argc, argv)) != NULL) { /* Load the input set */ if ((rv = skStreamCreate(&stream, SK_IO_READ, SK_CONTENT_SILK)) || (rv = skStreamBind(stream, filename)) || (rv = skStreamOpen(stream))) { skStreamLastErrMessage(stream, rv, errbuf, sizeof(errbuf)); skAppPrintErr("Unable to read IPset from '%s': %s", filename, errbuf); skStreamDestroy(&stream); continue; } rv = skIPSetRead(&input_set, stream); if (rv) { if (SKIPSET_ERR_FILEIO == rv) { skStreamLastErrMessage(stream, skStreamGetLastReturnValue(stream), errbuf, sizeof(errbuf)); } else { strncpy(errbuf, skIPSetStrerror(rv), sizeof(errbuf)); } skAppPrintErr("Unable to read IPset from '%s': %s", filename, errbuf); skStreamDestroy(&stream); continue; } skStreamDestroy(&stream); if (quiet || !count) { /* Only need to check for a match */ if (skIPSetCheckIPWildcard(input_set, &ipwild)) { found_match = 1; if (quiet) { goto done; } printf("%s\n", filename); } } else { /* Need a count of IPs, so intersect */ rv = skIPSetIntersect(input_set, wild_set); if (rv) { skAppPrintErr("Unable to intersect IPsets: %s", skIPSetStrerror(rv)); skIPSetDestroy(&input_set); skIPSetDestroy(&wild_set); return EXIT_FAILURE; } printf("%s:%s\n", filename, skIPSetCountIPsString(input_set, buf, sizeof(buf))); if ('0' != buf[0]) { found_match = 1; } } skIPSetDestroy(&input_set); } done: /* done */ skIPSetDestroy(&input_set); skIPSetDestroy(&wild_set); return ((found_match) ? 0 : 1); }