示例#1
0
/*
 *  appTeardown()
 *
 *    Teardown all modules, close all files, and tidy up all
 *    application state.
 *
 *    This function is idempotent.
 */
static void
appTeardown(
    void)
{
    static int teardown_flag = 0;

    if (teardown_flag) {
        return;
    }
    teardown_flag = 1;

    skAggBagDestroy(&ab);

    /* close output */
    skStreamClose(output);
    skStreamDestroy(&output);

    /* destroy string maps for keys and counters */
    skStringMapDestroy(key_name_map);
    key_name_map = NULL;

    skStringMapDestroy(counter_name_map);
    counter_name_map = NULL;

    skAggBagOptionsTeardown();
    skOptionsCtxDestroy(&optctx);
    skAppUnregister();
}
示例#2
0
int main(int argc, char **argv)
{
    int rv;

    appSetup(argc, argv);                 /* never returns on error */

    sortRandom();

    /* close the file */
    if ((rv = skStreamClose(out_rwios))
        || (rv = skStreamDestroy(&out_rwios)))
    {
        skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
        appExit(EXIT_FAILURE);
    }
    out_rwios = NULL;

    appExit(EXIT_SUCCESS);
    return 0; /* NOTREACHED */
}
示例#3
0
/*
 *  status = readerGetRecord(&out_rwrec, &out_probe, flow_processor);
 *
 *    Invoked by input_mode_type->get_record_fn();
 */
static fp_get_record_result_t
readerGetRecord(
    rwRec                  *out_rwrec,
    const skpc_probe_t    **out_probe,
    flow_proc_t            *fproc)
{
    static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
    skstream_t *fcfile;
    const char *filename;
    fp_get_record_result_t retVal = FP_GET_ERROR;
    int rv;

    pthread_mutex_lock(&mutex);

    /* If we don't have a source, get a file from the directory poller
     * and start processing it. */
    if (fproc->flow_src == NULL) {
        switch (readerGetNextValidFile(fproc)) {
          case 0:
            /* Success */
            break;
          case -1:
            /* Error getting file name (maybe in shutdown?) */
            goto END;
          case -2:  /* Error opening file */
          default:  /* Unexpected value */
            retVal = FP_FATAL_ERROR;
            goto END;
        }
    }
    fcfile = (skstream_t*)fproc->flow_src;

    /* Assume we can get a record from the probe. */
    retVal = FP_RECORD;
    *out_probe = fproc->probe;

    /* Try to get a record */
    rv = skStreamReadRecord(fcfile, out_rwrec);
    if (rv) {
        /* get failed: either at EOF or got an error. */
        if (rv != SKSTREAM_ERR_EOF) {
            skStreamPrintLastErr(fcfile, rv, &WARNINGMSG);
        }

        retVal = FP_FILE_BREAK;
        *out_probe = NULL;

        /* Print results for the file we just processed. */
        filename = skStreamGetPathname(fcfile);
        INFOMSG("Processed file %s, %" PRIu64 " records.",
                filename, skStreamGetRecordCount(fcfile));

        skStreamClose(fcfile);

        /* Either archive the file or remove it */
        archiveDirectoryInsertOrRemove(filename, NULL);

        /* All done with the flow source */
        skStreamDestroy(&fcfile);
        fproc->flow_src = NULL;
        fproc->probe = NULL;
    }

  END:
    pthread_mutex_unlock(&mutex);

    return retVal;
}
示例#4
0
/*
 *  mergeFiles(temp_file_idx)
 *
 *    Merge the temporary files numbered from 0 to 'temp_file_idx'
 *    inclusive into the output file 'out_ios', maintaining sorted
 *    order.  Exits the application if an error occurs.
 */
static void
mergeFiles(
    int                 temp_file_idx)
{
    char errbuf[2 * PATH_MAX];
    skstream_t *fps[MAX_MERGE_FILES];
    uint8_t recs[MAX_MERGE_FILES][NODE_SIZE];
    uint8_t lowest_rec[NODE_SIZE];
    int j;
    uint16_t open_count;
    uint16_t i;
    uint16_t lowest;
    uint16_t *top_heap;
    int tmp_idx_a;
    int tmp_idx_b;
    skstream_t *fp_intermediate = NULL;
    int tmp_idx_intermediate;
    skheap_t *heap;
    uint32_t heap_count;
    int opened_all_temps = 0;
    ssize_t rv;

    /* the index of the first temp file to the merge */
    tmp_idx_a = 0;

    TRACEMSG(("Merging #%d through #%d to '%s'",
              tmp_idx_a, temp_file_idx, skStreamGetPathname(out_rwios)));

    heap = skHeapCreate2(compHeapNodes, MAX_MERGE_FILES, sizeof(uint16_t),
                         NULL, recs);
    if (NULL == heap) {
        skAppPrintOutOfMemory("heap");
        appExit(EXIT_FAILURE);
    }

    /* This loop repeats as long as we haven't read all of the temp
     * files generated in the qsort stage. */
    do {
        assert(SKHEAP_ERR_EMPTY==skHeapPeekTop(heap,(skheapnode_t*)&top_heap));

        /* the index of the list temp file to merge */
        tmp_idx_b = temp_file_idx;

        /* open an intermediate temp file.  The merge-sort will have
         * to write records here if there are not enough file handles
         * available to open all the existing tempoary files. */
        fp_intermediate = skTempFileCreateStream(tmpctx, &tmp_idx_intermediate);
        if (fp_intermediate == NULL) {
            skAppPrintSyserror("Error creating new temporary file");
            appExit(EXIT_FAILURE);
        }

        /* count number of files we open */
        open_count = 0;

        /* Attempt to open up to MAX_MERGE_FILES, though we an open
         * may fail due to lack of resources (EMFILE or ENOMEM) */
        for (j = tmp_idx_a; j <= tmp_idx_b; ++j) {
            fps[open_count] = skTempFileOpenStream(tmpctx, j);
            if (fps[open_count] == NULL) {
                if ((open_count > 0)
                    && ((errno == EMFILE) || (errno == ENOMEM)))
                {
                    /* We cannot open any more files.  Rewind counter
                     * by one to catch this file on the next merge */
                    tmp_idx_b = j - 1;
                    TRACEMSG((("FILE limit hit--"
                               "merging #%d through #%d into #%d: %s"),
                              tmp_idx_a, tmp_idx_b, tmp_idx_intermediate,
                              strerror(errno)));
                    break;
                } else {
                    skAppPrintSyserror(("Error opening existing"
                                        " temporary file '%s'"),
                                       skTempFileGetName(tmpctx, j));
                    appExit(EXIT_FAILURE);
                }
            }

            /* read the first record */
            rv = skStreamRead(fps[open_count], recs[open_count], NODE_SIZE);
            if (NODE_SIZE == rv) {
                /* insert the file index into the heap */
                skHeapInsert(heap, &open_count);
                ++open_count;
                if (open_count == MAX_MERGE_FILES) {
                    /* We've reached the limit for this pass.  Set
                     * tmp_idx_b to the file we just opened. */
                    tmp_idx_b = j;
                    TRACEMSG((("MAX_MERGE_FILES limit hit--"
                           "merging #%d through #%d to #%d"),
                              tmp_idx_a, tmp_idx_b, tmp_idx_intermediate));
                    break;
                }
            } else if (0 == rv) {
                TRACEMSG(("Ignoring empty temporary file '%s'",
                          skTempFileGetName(tmpctx, j)));
                skStreamDestroy(&fps[open_count]);
            } else {
                if (rv > 0) {
                    snprintf(errbuf, sizeof(errbuf),
                             "Short read %" SK_PRIdZ "/%" PRIu32 " from '%s'",
                             rv, NODE_SIZE,
                             skStreamGetPathname(fps[open_count]));
                } else {
                    skStreamLastErrMessage(
                        fps[open_count], rv, errbuf, sizeof(errbuf));
                }
                skAppPrintErr(
                    "Error reading first record from temporary file: %s",
                    errbuf);
                appExit(EXIT_FAILURE);
            }
        }

        /* Here, we check to see if we've opened all temp files.  If
         * so, set a flag so we write data to final destination and
         * break out of the loop after we're done. */
        if (tmp_idx_b == temp_file_idx) {
            opened_all_temps = 1;
            /* no longer need the intermediate temp file */
            skStreamDestroy(&fp_intermediate);
        } else {
            /* we could not open all temp files, so merge all opened
             * temp files into the intermediate file.  Add the
             * intermediate file to the list of files to merge */
            temp_file_idx = tmp_idx_intermediate;
        }

        TRACEMSG((("Merging %" PRIu16 " temporary files"), open_count));

        heap_count = skHeapGetNumberEntries(heap);
        assert(heap_count == open_count);

        /* get the index of the file with the lowest record; which is
         * at the top of the heap */
        if (skHeapPeekTop(heap, (skheapnode_t*)&top_heap) != SKHEAP_OK) {
            skAppPrintErr("Unable to open and read any temporary files.");
            appExit(EXIT_FAILURE);
        }
        lowest = *top_heap;

        /* exit this do...while() once all records for all opened
         * files have been read */
        do {
            /* lowest_rec is the record pointed to by the index at the
             * top of the heap */
            memcpy(lowest_rec, recs[lowest], NODE_SIZE);

            /* write the record */
            if (fp_intermediate) {
                /* write the record to intermediate tmp file */
                rv = skStreamWrite(fp_intermediate, lowest_rec, NODE_SIZE);
                if (NODE_SIZE != rv) {
                    skAppPrintSyserror(
                        "Error writing record to temporary file '%s'",
                        skTempFileGetName(tmpctx, tmp_idx_intermediate));
                    appExit(EXIT_FAILURE);
                }
            } else {
                /* we successfully opened all (remaining) temp files,
                 * write to record to the final destination */
                rv = skStreamWriteRecord(out_rwios, (rwRec*)lowest_rec);
                if (0 != rv) {
                    skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
                    if (SKSTREAM_ERROR_IS_FATAL(rv)) {
                        appExit(EXIT_FAILURE);
                    }
                }
            }

            /* replace the record we just processed and loop over all
             * files until we get a record that is not a duplicate */
            do {
                if ((rv = skStreamRead(fps[lowest], recs[lowest], NODE_SIZE))
                    != NODE_SIZE)
                {
                    /* read failed.  there is no more data for this
                     * file; remove it from the heap; if the heap is
                     * empty, exit the loop */
                    skHeapExtractTop(heap, NULL);
                    --heap_count;
#if TRACEMSG_LEVEL > 0
                    if (rv == 0) {
                        TRACEMSG(
                            ("Finished reading file #%u: EOF; %u files remain",
                             (tmp_idx_a + lowest), heap_count));
                    } else if (rv > 0) {
                        TRACEMSG(
                            ("Finished reading file #%u: Short read "
                             "%" SK_PRIdZ "/%" PRIu32 "; %u files remain",
                             tmp_idx_a + lowest, rv, NODE_SIZE, heap_count));
                    } else {
                        skStreamLastErrMessage(
                            fps[open_count], rv, errbuf, sizeof(errbuf));
                        TRACEMSG(
                            ("Finished reading file #%u: %s; %u files remain",
                             (tmp_idx_a + lowest), errbuf, heap_count));
                    }
#endif  /* TRACEMSG_LEVEL */
                    if (0 == heap_count) {
                        break;
                    }

                } else if (rwrecCompare(lowest_rec, recs[lowest])) {
                    /* read succeeded.  new record is not a
                     * duplicate and we insert it into the heap */
                    /* FIXME: This comparison reduces work when the
                     * keys are the same, but it adds another
                     * comparison when the keys are different; is this
                     * an overall win or lose? */
                    skHeapReplaceTop(heap, &lowest, NULL);

                } else {
                    /* read succeeded.  record is a duplicate; ignore
                     * the record and leave the heap unchanged */
                    continue;
                }

                /* get the record at the top of the heap and see if it
                 * is a duplicate; if it is, ignore it. */
                skHeapPeekTop(heap, (skheapnode_t*)&top_heap);
                lowest = *top_heap;
            } while (0 == rwrecCompare(lowest_rec, recs[lowest]));
        } while (heap_count > 0);

        TRACEMSG((("Finished processing #%d through #%d"),
                  tmp_idx_a, tmp_idx_b));

        /* Close all open temp files */
        for (i = 0; i < open_count; ++i) {
            skStreamDestroy(&fps[i]);
        }
        /* Delete all temp files we opened (or attempted to open) this
         * time */
        for (j = tmp_idx_a; j <= tmp_idx_b; ++j) {
            skTempFileRemove(tmpctx, j);
        }

        /* Close the intermediate temp file. */
        if (fp_intermediate) {
            rv = skStreamClose(fp_intermediate);
            if (rv) {
                skStreamLastErrMessage(
                    fp_intermediate, rv, errbuf, sizeof(errbuf));
                skAppPrintErr("Error closing temporary file: %s", errbuf);
                skStreamDestroy(&fp_intermediate);
                appExit(EXIT_FAILURE);
            }
            skStreamDestroy(&fp_intermediate);
        }

        /* Start the next merge with the next input temp file */
        tmp_idx_a = tmp_idx_b + 1;

    } while (!opened_all_temps);

    skHeapFree(heap);
}