/*
* Parse the argument to the --log-flags switch.
*/
static int
parseLogFlags(
const char *log_flags_str)
{
char *log_flags_copy = NULL;
char *flag_next;
char *flag;
int rv = -1;
skpcProbeClearLogFlags(probe);
if (NULL == log_flags_str) {
return 0;
}
/* create a copy of the input string and maintain a reference to
* it so we can free it */
log_flags_copy = strdup(log_flags_str);
flag_next = log_flags_copy;
if (NULL == log_flags_copy) {
skAppPrintOutOfMemory(NULL);
goto END;
}
/* parse the flags as a comma separated list of tokens */
while ((flag = strsep(&flag_next, ",")) != NULL) {
/* check for empty token (e.g., double comma) */
if ('\0' == *flag) {
continue;
}
switch (skpcProbeAddLogFlag(probe, flag)) {
case 0:
break;
case -1:
skAppPrintErr("Invalid %s: Unrecognized value '%s'",
appOptions[OPT_LOG_FLAGS].name, flag);
goto END;
case -2:
skAppPrintErr("Invalid %s: Cannot mix 'none' with other value",
appOptions[OPT_LOG_FLAGS].name);
goto END;
default:
skAppPrintErr("Bad return value from skpcProbeAddLogFlag()");
skAbort();
}
}
rv = 0;
END:
free(log_flags_copy);
return rv;
}
/*
* status = appOptionsHandler(cData, opt_index, opt_arg);
*
* This function is passed to skOptionsRegister(); it will be called
* by skOptionsParse() for each user-specified switch that the
* application has registered; it should handle the switch as
* required---typically by setting global variables---and return 1
* if the switch processing failed or 0 if it succeeded. Returning
* a non-zero from from the handler causes skOptionsParse() to return
* a negative value.
*
* The clientData in 'cData' is typically ignored; 'opt_index' is
* the index number that was specified as the last value for each
* struct option in appOptions[]; 'opt_arg' is the user's argument
* to the switch for options that have a REQUIRED_ARG or an
* OPTIONAL_ARG.
*/
static int
appOptionsHandler(
clientData UNUSED(cData),
int opt_index,
char *opt_arg)
{
int rv;
switch ((appOptionsEnum)opt_index) {
case OPT_DESTINATION_DIR:
if (skOptionsCheckDirectory(opt_arg, appOptions[opt_index].name)) {
return 1;
}
destination_dir = opt_arg;
break;
case OPT_DUPLICATE_DEST:
if (skOptionsCheckDirectory(opt_arg, appOptions[opt_index].name)) {
return 1;
}
rv = skDLListPushTail(duplicate_dirs, opt_arg);
if (rv != 0) {
skAppPrintOutOfMemory("directory name");
return 1;
}
break;
case OPT_UNIQUE_DUPLICATES:
unique_duplicates = 1;
break;
case OPT_POST_COMMAND:
if (verifyCommandString(opt_arg)) {
return 1;
}
post_command = opt_arg;
break;
#ifdef SK_HAVE_STATVFS
case OPT_FREESPACE_MINIMUM:
{
uint64_t tmp_64;
rv = skStringParseHumanUint64(&tmp_64, opt_arg, SK_HUMAN_NORMAL);
if (rv) {
goto PARSE_ERROR;
}
freespace_minimum = (int64_t)tmp_64;
}
break;
case OPT_SPACE_MAXIMUM_PERCENT:
rv = skStringParseDouble(&space_maximum_percent, opt_arg, 0.0, 100.0);
if (rv) {
goto PARSE_ERROR;
}
break;
#endif /* SK_HAVE_STATVFS */
}
return 0; /* OK */
#ifdef SK_HAVE_STATVFS
PARSE_ERROR:
skAppPrintErr("Invalid %s '%s': %s",
appOptions[opt_index].name, opt_arg,
skStringParseStrerror(rv));
return 1;
#endif
}
/*
* appSetup(argc, argv);
*
* Perform all the setup for this application include setting up
* required modules, parsing options, etc. This function should be
* passed the same arguments that were passed into main().
*
* Returns to the caller if all setup succeeds. If anything fails,
* this function will cause the application to exit with a FAILURE
* exit status.
*/
static void
appSetup(
int argc,
char **argv)
{
SILK_FEATURES_DEFINE_STRUCT(features);
int arg_index;
int rv;
/* check that we have the same number of options entries and help*/
assert((sizeof(appHelp)/sizeof(char *)) ==
(sizeof(appOptions)/sizeof(struct option)));
/* register the application */
skAppRegister(argv[0]);
skAppVerifyFeatures(&features, NULL);
skOptionsSetUsageCallback(&appUsageLong);
/* initialize globals */
shuttingdown = 0;
destination_dir = NULL;
duplicate_dirs = skDLListCreate(NULL);
if (duplicate_dirs == NULL) {
skAppPrintOutOfMemory("duplicate directory list");
exit(EXIT_FAILURE);
}
open_file_list = skDLListCreate(NULL);
if (open_file_list == NULL) {
skAppPrintOutOfMemory("open file list");
exit(EXIT_FAILURE);
}
transfers = transferIdentTreeCreate();
if (transfers == NULL) {
skAppPrintOutOfMemory("receiver data structure");
exit(EXIT_FAILURE);
}
#ifdef SK_HAVE_STATVFS
{
uint64_t tmp_64;
rv = skStringParseHumanUint64(&tmp_64, DEFAULT_FREESPACE_MINIMUM,
SK_HUMAN_NORMAL);
if (rv) {
skAppPrintErr("Bad default value for freespace_minimum: '%s': %s",
DEFAULT_FREESPACE_MINIMUM,skStringParseStrerror(rv));
exit(EXIT_FAILURE);
}
freespace_minimum = (int64_t)tmp_64;
}
#endif /* SK_HAVE_STATVFS */
/* register the options and handler */
if (skOptionsRegister(appOptions, &appOptionsHandler, NULL))
{
skAppPrintErr("Unable to register application options");
exit(EXIT_FAILURE);
}
/* Register the other transfer options */
if (transferSetup()) {
exit(EXIT_FAILURE);
}
/* rwreceiver runs as a daemon */
if (skdaemonSetup((SKLOG_FEATURE_LEGACY | SKLOG_FEATURE_SYSLOG),
argc, argv))
{
exit(EXIT_FAILURE);
}
/* register the teardown handler */
if (atexit(appTeardown) < 0) {
skAppPrintErr("Unable to register appTeardown() with atexit()");
appTeardown();
exit(EXIT_FAILURE);
}
/* parse the options */
arg_index = skOptionsParse(argc, argv);
if (arg_index < 0) {
/* options parsing should print error */
skAppUsage(); /* never returns */
}
/* Verify the options */
rv = rwreceiverVerifyOptions();
/* verify the required options for logging */
if (skdaemonOptionsVerify()) {
rv = -1;
}
if (rv) {
skAppUsage(); /* never returns */
}
/* check for extraneous arguments */
if (arg_index != argc) {
skAppPrintErr("Too many arguments or unrecognized switch --%s",
argv[arg_index]);
skAppUsage(); /* never returns */
}
/* Identify the main thread */
skthread_init("main");
return; /* OK */
}
/*
* mergeFiles(temp_file_idx)
*
* Merge the temporary files numbered from 0 to 'temp_file_idx'
* inclusive into the output file 'out_ios', maintaining sorted
* order. Exits the application if an error occurs.
*/
static void
mergeFiles(
int temp_file_idx)
{
char errbuf[2 * PATH_MAX];
skstream_t *fps[MAX_MERGE_FILES];
uint8_t recs[MAX_MERGE_FILES][NODE_SIZE];
uint8_t lowest_rec[NODE_SIZE];
int j;
uint16_t open_count;
uint16_t i;
uint16_t lowest;
uint16_t *top_heap;
int tmp_idx_a;
int tmp_idx_b;
skstream_t *fp_intermediate = NULL;
int tmp_idx_intermediate;
skheap_t *heap;
uint32_t heap_count;
int opened_all_temps = 0;
ssize_t rv;
/* the index of the first temp file to the merge */
tmp_idx_a = 0;
TRACEMSG(("Merging #%d through #%d to '%s'",
tmp_idx_a, temp_file_idx, skStreamGetPathname(out_rwios)));
heap = skHeapCreate2(compHeapNodes, MAX_MERGE_FILES, sizeof(uint16_t),
NULL, recs);
if (NULL == heap) {
skAppPrintOutOfMemory("heap");
appExit(EXIT_FAILURE);
}
/* This loop repeats as long as we haven't read all of the temp
* files generated in the qsort stage. */
do {
assert(SKHEAP_ERR_EMPTY==skHeapPeekTop(heap,(skheapnode_t*)&top_heap));
/* the index of the list temp file to merge */
tmp_idx_b = temp_file_idx;
/* open an intermediate temp file. The merge-sort will have
* to write records here if there are not enough file handles
* available to open all the existing tempoary files. */
fp_intermediate = skTempFileCreateStream(tmpctx, &tmp_idx_intermediate);
if (fp_intermediate == NULL) {
skAppPrintSyserror("Error creating new temporary file");
appExit(EXIT_FAILURE);
}
/* count number of files we open */
open_count = 0;
/* Attempt to open up to MAX_MERGE_FILES, though we an open
* may fail due to lack of resources (EMFILE or ENOMEM) */
for (j = tmp_idx_a; j <= tmp_idx_b; ++j) {
fps[open_count] = skTempFileOpenStream(tmpctx, j);
if (fps[open_count] == NULL) {
if ((open_count > 0)
&& ((errno == EMFILE) || (errno == ENOMEM)))
{
/* We cannot open any more files. Rewind counter
* by one to catch this file on the next merge */
tmp_idx_b = j - 1;
TRACEMSG((("FILE limit hit--"
"merging #%d through #%d into #%d: %s"),
tmp_idx_a, tmp_idx_b, tmp_idx_intermediate,
strerror(errno)));
break;
} else {
skAppPrintSyserror(("Error opening existing"
" temporary file '%s'"),
skTempFileGetName(tmpctx, j));
appExit(EXIT_FAILURE);
}
}
/* read the first record */
rv = skStreamRead(fps[open_count], recs[open_count], NODE_SIZE);
if (NODE_SIZE == rv) {
/* insert the file index into the heap */
skHeapInsert(heap, &open_count);
++open_count;
if (open_count == MAX_MERGE_FILES) {
/* We've reached the limit for this pass. Set
* tmp_idx_b to the file we just opened. */
tmp_idx_b = j;
TRACEMSG((("MAX_MERGE_FILES limit hit--"
"merging #%d through #%d to #%d"),
tmp_idx_a, tmp_idx_b, tmp_idx_intermediate));
break;
}
} else if (0 == rv) {
TRACEMSG(("Ignoring empty temporary file '%s'",
skTempFileGetName(tmpctx, j)));
skStreamDestroy(&fps[open_count]);
} else {
if (rv > 0) {
snprintf(errbuf, sizeof(errbuf),
"Short read %" SK_PRIdZ "/%" PRIu32 " from '%s'",
rv, NODE_SIZE,
skStreamGetPathname(fps[open_count]));
} else {
skStreamLastErrMessage(
fps[open_count], rv, errbuf, sizeof(errbuf));
}
skAppPrintErr(
"Error reading first record from temporary file: %s",
errbuf);
appExit(EXIT_FAILURE);
}
}
/* Here, we check to see if we've opened all temp files. If
* so, set a flag so we write data to final destination and
* break out of the loop after we're done. */
if (tmp_idx_b == temp_file_idx) {
opened_all_temps = 1;
/* no longer need the intermediate temp file */
skStreamDestroy(&fp_intermediate);
} else {
/* we could not open all temp files, so merge all opened
* temp files into the intermediate file. Add the
* intermediate file to the list of files to merge */
temp_file_idx = tmp_idx_intermediate;
}
TRACEMSG((("Merging %" PRIu16 " temporary files"), open_count));
heap_count = skHeapGetNumberEntries(heap);
assert(heap_count == open_count);
/* get the index of the file with the lowest record; which is
* at the top of the heap */
if (skHeapPeekTop(heap, (skheapnode_t*)&top_heap) != SKHEAP_OK) {
skAppPrintErr("Unable to open and read any temporary files.");
appExit(EXIT_FAILURE);
}
lowest = *top_heap;
/* exit this do...while() once all records for all opened
* files have been read */
do {
/* lowest_rec is the record pointed to by the index at the
* top of the heap */
memcpy(lowest_rec, recs[lowest], NODE_SIZE);
/* write the record */
if (fp_intermediate) {
/* write the record to intermediate tmp file */
rv = skStreamWrite(fp_intermediate, lowest_rec, NODE_SIZE);
if (NODE_SIZE != rv) {
skAppPrintSyserror(
"Error writing record to temporary file '%s'",
skTempFileGetName(tmpctx, tmp_idx_intermediate));
appExit(EXIT_FAILURE);
}
} else {
/* we successfully opened all (remaining) temp files,
* write to record to the final destination */
rv = skStreamWriteRecord(out_rwios, (rwRec*)lowest_rec);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
appExit(EXIT_FAILURE);
}
}
}
/* replace the record we just processed and loop over all
* files until we get a record that is not a duplicate */
do {
if ((rv = skStreamRead(fps[lowest], recs[lowest], NODE_SIZE))
!= NODE_SIZE)
{
/* read failed. there is no more data for this
* file; remove it from the heap; if the heap is
* empty, exit the loop */
skHeapExtractTop(heap, NULL);
--heap_count;
#if TRACEMSG_LEVEL > 0
if (rv == 0) {
TRACEMSG(
("Finished reading file #%u: EOF; %u files remain",
(tmp_idx_a + lowest), heap_count));
} else if (rv > 0) {
TRACEMSG(
("Finished reading file #%u: Short read "
"%" SK_PRIdZ "/%" PRIu32 "; %u files remain",
tmp_idx_a + lowest, rv, NODE_SIZE, heap_count));
} else {
skStreamLastErrMessage(
fps[open_count], rv, errbuf, sizeof(errbuf));
TRACEMSG(
("Finished reading file #%u: %s; %u files remain",
(tmp_idx_a + lowest), errbuf, heap_count));
}
#endif /* TRACEMSG_LEVEL */
if (0 == heap_count) {
break;
}
} else if (rwrecCompare(lowest_rec, recs[lowest])) {
/* read succeeded. new record is not a
* duplicate and we insert it into the heap */
/* FIXME: This comparison reduces work when the
* keys are the same, but it adds another
* comparison when the keys are different; is this
* an overall win or lose? */
skHeapReplaceTop(heap, &lowest, NULL);
} else {
/* read succeeded. record is a duplicate; ignore
* the record and leave the heap unchanged */
continue;
}
/* get the record at the top of the heap and see if it
* is a duplicate; if it is, ignore it. */
skHeapPeekTop(heap, (skheapnode_t*)&top_heap);
lowest = *top_heap;
} while (0 == rwrecCompare(lowest_rec, recs[lowest]));
} while (heap_count > 0);
TRACEMSG((("Finished processing #%d through #%d"),
tmp_idx_a, tmp_idx_b));
/* Close all open temp files */
for (i = 0; i < open_count; ++i) {
skStreamDestroy(&fps[i]);
}
/* Delete all temp files we opened (or attempted to open) this
* time */
for (j = tmp_idx_a; j <= tmp_idx_b; ++j) {
skTempFileRemove(tmpctx, j);
}
/* Close the intermediate temp file. */
if (fp_intermediate) {
rv = skStreamClose(fp_intermediate);
if (rv) {
skStreamLastErrMessage(
fp_intermediate, rv, errbuf, sizeof(errbuf));
skAppPrintErr("Error closing temporary file: %s", errbuf);
skStreamDestroy(&fp_intermediate);
appExit(EXIT_FAILURE);
}
skStreamDestroy(&fp_intermediate);
}
/* Start the next merge with the next input temp file */
tmp_idx_a = tmp_idx_b + 1;
} while (!opened_all_temps);
skHeapFree(heap);
}
/*
* status = timestampFormatParse(format_string, out_flags);
*
* Parse the comma-separated list of timestamp format strings
* contained in 'format_string' and set 'out_flags' to the result
* of parsing the string. Return 0 on success, or -1 if parsing of
* the values fails or conflicting values are given.
*/
static int
timestampFormatParse(
const char *format,
uint32_t *out_flags)
{
char buf[256];
char *errmsg;
sk_stringmap_t *str_map = NULL;
sk_stringmap_iter_t *iter = NULL;
sk_stringmap_entry_t *found_entry;
const sk_stringmap_entry_t *entry;
int name_seen = 0;
int zone_seen = 0;
int rv = -1;
/* create a stringmap of the available timestamp formats */
if (SKSTRINGMAP_OK != skStringMapCreate(&str_map)) {
skAppPrintOutOfMemory(NULL);
goto END;
}
if (skStringMapAddEntries(str_map, -1, timestamp_names) != SKSTRINGMAP_OK){
skAppPrintOutOfMemory(NULL);
goto END;
}
if (skStringMapAddEntries(str_map, -1, timestamp_zones) != SKSTRINGMAP_OK){
skAppPrintOutOfMemory(NULL);
goto END;
}
/* attempt to match */
if (skStringMapParse(str_map, format, SKSTRINGMAP_DUPES_ERROR,
&iter, &errmsg))
{
skAppPrintErr("Invalid %s: %s",
appOptions[OPT_TIMESTAMP_FORMAT].name, errmsg);
goto END;
}
*out_flags = 0;
while (skStringMapIterNext(iter, &found_entry, NULL) == SK_ITERATOR_OK) {
*out_flags |= found_entry->id;
switch (found_entry->id) {
#if 0
case SKTIMESTAMP_NOMSEC:
break;
#endif
case 0:
case SKTIMESTAMP_EPOCH:
case SKTIMESTAMP_ISO:
case SKTIMESTAMP_MMDDYYYY:
if (name_seen) {
entry = timestamp_names;
strncpy(buf, entry->name, sizeof(buf));
for (++entry; entry->name; ++entry) {
strncat(buf, ",", sizeof(buf)-strlen(buf)-1);
strncat(buf, entry->name, sizeof(buf)-strlen(buf)-1);
}
skAppPrintErr("Invalid %s: May only specify one of %s",
appOptions[OPT_TIMESTAMP_FORMAT].name, buf);
goto END;
}
name_seen = 1;
break;
case SKTIMESTAMP_UTC:
case SKTIMESTAMP_LOCAL:
if (zone_seen) {
entry = timestamp_zones;
strncpy(buf, entry->name, sizeof(buf));
for (++entry; entry->name; ++entry) {
strncat(buf, ",", sizeof(buf)-strlen(buf)-1);
strncat(buf, entry->name, sizeof(buf)-strlen(buf)-1);
}
skAppPrintErr("Invalid %s: May only specify one of %s",
appOptions[OPT_TIMESTAMP_FORMAT].name, buf);
goto END;
}
zone_seen = 1;
break;
default:
skAbortBadCase(found_entry->id);
}
}
rv = 0;
END:
if (str_map) {
skStringMapDestroy(str_map);
}
if (iter) {
skStringMapIterDestroy(iter);
}
return rv;
}
/*
* status = loadschemeParse(scheme_name, load_scheme);
*
* Parse the load-scheme name in 'scheme_name' and set
* 'load_scheme' to the result of parsing the string. Return 0 on
* success, or -1 if parsing of the value fails.
*/
static int
loadschemeParse(
const char *scheme_name,
bin_load_scheme_enum_t *load_scheme)
{
char buf[128];
sk_stringmap_entry_t new_entry;
sk_stringmap_t *str_map = NULL;
sk_stringmap_status_t sm_err;
sk_stringmap_entry_t *sm_entry;
const sk_stringmap_entry_t *e;
int rv = -1;
memset(&new_entry, 0, sizeof(new_entry));
new_entry.name = buf;
/* create a stringmap of the available load-scheme names */
if (SKSTRINGMAP_OK != skStringMapCreate(&str_map)) {
skAppPrintOutOfMemory(NULL);
goto END;
}
if (skStringMapAddEntries(str_map, -1, load_schemes) != SKSTRINGMAP_OK){
skAppPrintOutOfMemory(NULL);
goto END;
}
/* allow the integer ID of each load-scheme to work */
for (e = load_schemes; e->name; ++e) {
new_entry.id = e->id;
snprintf(buf, sizeof(buf), "%u", e->id);
if (skStringMapAddEntries(str_map, 1, &new_entry) != SKSTRINGMAP_OK) {
skAppPrintOutOfMemory(NULL);
goto END;
}
}
/* attempt to match */
sm_err = skStringMapGetByName(str_map, scheme_name, &sm_entry);
switch (sm_err) {
case SKSTRINGMAP_OK:
*load_scheme = (bin_load_scheme_enum_t)sm_entry->id;
rv = 0;
break;
case SKSTRINGMAP_PARSE_AMBIGUOUS:
skAppPrintErr("Invalid %s: '%s' is ambiguous",
appOptions[OPT_LOAD_SCHEME].name, scheme_name);
break;
case SKSTRINGMAP_PARSE_NO_MATCH:
skAppPrintErr("Invalid %s: '%s' is not recognized",
appOptions[OPT_LOAD_SCHEME].name, scheme_name);
break;
default:
skAppPrintErr("Unexpected return value from string-map parser (%d)",
sm_err);
break;
}
END:
if (str_map) {
skStringMapDestroy(str_map);
}
return rv;
}
