static void logToAllLevels( void *v_name) { #define ST_ND_TH(c) \ ((((c) % 10) == 1) ? "st" : ((((c) % 10) == 2) ? "nd" : "th")) int c; char *name = (char *)v_name; pthread_mutex_lock(&count_mutex); c = ++global_count; pthread_mutex_unlock(&count_mutex); if (test_emerg) { EMERGMSG("Writing a EMERGMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); } ALERTMSG("Writing a ALERTMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); CRITMSG("Writing a CRITMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); ERRMSG("Writing a ERRMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); WARNINGMSG("Writing a WARNINGMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); NOTICEMSG("Writing a NOTICEMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); INFOMSG("Writing a INFOMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); DEBUGMSG("Writing a DEBUGMSG for the %d%s time [%s]", c, ST_ND_TH(c), name); }
/* * status = readerStart(flow_processor); * * Invoked by input_mode_type->start_fn(); */ static int readerStart( flow_proc_t UNUSED(*fproc)) { /* Create the polldir object for directory polling */ INFOMSG(("Creating " INPUT_MODE_TYPE_NAME " directory poller for '%s'"), incoming_directory); polldir = skPollDirCreate(incoming_directory, polling_interval); if (NULL == polldir) { CRITMSG("Could not initiate polling for %s", incoming_directory); return 1; } return 0; }
/* * readerGetNextValidFile(&fc_src); * * Pull the next file name off of the valid-queue and create a * flowsource object to read the flowcap records in it. Fills * 'fproc' with the new flowcap-source object and probe. * * Return 0 on success. Return -1 if getting the file name fails. * If unable to open the file or file not of correct form, return * -2 unless the --error-dir is set, in which case move the file * there and try the next file. */ static int readerGetNextValidFile( flow_proc_t *fproc) { skstream_t *fcfile = NULL; skpc_probe_t *probe = NULL; skPollDirErr_t pderr; char *filename; char path[PATH_MAX]; int rv; do { /* Get next file from the directory poller */ pderr = skPollDirGetNextFile(polldir, path, &filename); if (pderr != PDERR_NONE) { if (pderr == PDERR_STOPPED) { return -1; } CRITMSG("Fatal polldir error ocurred: %s", ((pderr == PDERR_SYSTEM) ? strerror(errno) : skPollDirStrError(pderr))); skAbort(); } INFOMSG((INPUT_MODE_TYPE_NAME " processing %s"), filename); /* open the file to create a source of records */ rv = flowcapSourceCreateFromFile(path, &fcfile, &probe); if (rv) { rv = errorDirectoryInsertFile(path); if (rv != 0) { /* either no --error-dir (rv == 1) or problem moving * the file (rv == -1). either way, return an error * code to the caller. */ return -2; } } } while (fcfile == NULL); fproc->flow_src = fcfile; fproc->probe = probe; return 0; }
/* * Helper function that opens the flowcap file at 'path'. The * handle to the file is put in the location specified by 'stream'. * The name of the probe is read from the file's header, and the * probe object is put into 'probe'. Returns 0 on success, or -1 * on the following error conditions: * * -- 'path' is NULL. * -- unable to open file * -- file is not a valid SiLK file * -- file does not contain a Probename Header * -- the probe_name does not map to a valid probe in probeconf * * NOTES: * * Flowcap V16 files have the probe name in the header. * * Flowcap Files V2-V5 have the sensor name and probe name in the * header. When these are processed in SiLK 1.0, these get mapped * to the probe name "<sensor>_<probe>". * * Flowcap Files V1 have no probe information and are no longer * supported. */ static int flowcapSourceCreateFromFile( const char *path, skstream_t **stream, skpc_probe_t **probe) { sk_file_header_t *hdr; sk_header_entry_t *sp_hdr; const char *probe_name; int rv; /* Argument checking */ if (path == NULL) { ERRMSG("NULL path passed to flowcapSourceCreateFromFile."); return -1; } /* Valid file checking */ rv = skStreamOpenSilkFlow(stream, path, SK_IO_READ); if (rv) { CRITMSG("Unable to open '%s' for reading.", path); skStreamPrintLastErr(*stream, rv, &ERRMSG); goto error; } /* * File should have a Probename header * * Flowcap V16 files have the probe name in the header. * * Flowcap Files V2-V5 have a separate sensor name and probe name * in the header. In SiLK 1.0, these get mapped to the single * probe name "<sensor>_<probe>". * * Flowcap Files V1 have no probe information and are not * supported. */ hdr = skStreamGetSilkHeader(*stream); sp_hdr = skHeaderGetFirstMatch(hdr, SK_HENTRY_PROBENAME_ID); if (sp_hdr == NULL) { CRITMSG("No probename header in %s.", path); goto error; } probe_name = skHentryProbenameGetProbeName(sp_hdr); if (probe_name == NULL || probe_name[0] == '\0') { CRITMSG("Unable to get probename from flowcap file '%s'.", path); goto error; } /* Use the probe_name to find the skpc_probe_t object. */ *probe = (skpc_probe_t*)skpcProbeLookupByName(probe_name); if (*probe == NULL) { CRITMSG("The sensor configuration file does not define probe '%s'", probe_name); goto error; } /* Verify that the probe has sensors associated with it */ if (skpcProbeGetSensorCount(*probe) == 0) { CRITMSG("Probe '%s' is not associated with a sensor", probe_name); goto error; } /* File has been validated. We're done. */ return 0; error: skStreamDestroy(stream); return -1; }
/* * This function is called by the handleConnection() function in * rwtransfer.c once the connection has been established. This * function returns -1 on error, 0 if no files were transferred, or * 1 if one or more files were successfully received. */ int transferFiles( sk_msg_queue_t *q, skm_channel_t channel, transfer_t *sndr) { static pthread_mutex_t open_file_mutex = PTHREAD_MUTEX_INITIALIZER; int fd = -1; uint64_t size = 0; uint64_t pa_size = 0; uint8_t *map = NULL; char *name = NULL; char *dotname = NULL; char dotpath[PATH_MAX]; char destpath[sizeof(dotpath)-1]; struct stat st; ino_t *inode; int proto_err; int rv; sk_dll_iter_t iter; const char *duplicate_dir; enum transfer_state {File_info, File_info_ack, Send_file, Complete_ack, Error} state; int thread_exit; int transferred_file = 0; state = File_info; proto_err = 0; thread_exit = 0; destpath[0] = '\0'; dotpath[0] = '\0'; memset(&st, 0, sizeof(st)); while (!shuttingdown && !proto_err && !thread_exit && !sndr->disconnect && (state != Error)) { sk_msg_t *msg; /* Handle reads */ switch (state) { case File_info: case Send_file: rv = skMsgQueueGetMessage(q, &msg); if (rv == -1) { ASSERT_ABORT(shuttingdown); continue; } if (handleDisconnect(msg, sndr->ident)) { state = Error; } break; case Error: ASSERT_ABORT(0); break; default: msg = NULL; } /* Handle all states */ switch (state) { case File_info: /* Create the placeholder and dot files and mmap() the * space. */ { file_info_t *finfo; uint32_t len; mode_t mode; off_t offrv; if ((proto_err = checkMsg(msg, q, CONN_NEW_FILE))) { break; } DEBUG_PRINT1("Received CONN_NEW_FILE"); finfo = (file_info_t *)skMsgMessage(msg); size = (uint64_t)ntohl(finfo->high_filesize) << 32 | ntohl(finfo->low_filesize); pa_size = size; /* blocksize = ntohl(finfo->block_size); --- UNUSED */ mode = ntohl(finfo->mode) & 0777; len = skMsgLength(msg) - offsetof(file_info_t, filename); dotname = (char *)calloc(1, len + 1); CHECK_ALLOC(dotname); name = dotname + 1; dotname[0] = '.'; memcpy(name, finfo->filename, len); if (!memchr(name, '\0', len)) { sendString(q, channel, EXTERNAL, SEND_CONN_REJECT(sndr), LOG_WARNING, "Illegal filename (from %s)", sndr->ident); state = FILE_INFO_ERROR_STATE(sndr); break; } INFOMSG("Receiving from %s: '%s' (%" PRIu64 " bytes)", sndr->ident, name, size); /* Check filesystem for enough space for file */ if (CHECK_DISK_SPACE(pa_size)) { WARNINGMSG(("Not enough space on filesystem for %" PRIu64 " byte file '%s'"), pa_size, name); pa_size = 0; state = FILESYSTEM_FULL_ERROR_STATE(sndr); break; } /* Create the placeholder file */ rv = snprintf(destpath, sizeof(destpath), "%s/%s", destination_dir, name); if ((size_t)rv >= sizeof(destpath)) { sendString(q, channel, EXTERNAL, SEND_CONN_REJECT(sndr), LOG_WARNING, "Filename too long (from %s)", sndr->ident); state = FILE_INFO_ERROR_STATE(sndr); destpath[0] = '\0'; break; } assert((size_t)rv < sizeof(destpath)); pthread_mutex_lock(&open_file_mutex); reopen: fd = open(destpath, O_CREAT | O_EXCL | O_WRONLY, 0); if (fd == -1) { if (errno != EEXIST) { CRITMSG("Could not create '%s': %s", destpath, strerror(errno)); thread_exit = 1; pthread_mutex_unlock(&open_file_mutex); break; } if (stat(destpath, &st) == -1) { WARNINGMSG("Unable to stat '%s': %s", destpath, strerror(errno)); } else if (S_ISREG(st.st_mode) && ((st.st_mode & 0777) == 0) && ((st.st_size == 0))) { /* looks like a placeholder file. are we * receiving a file with the same name from a * different rwsender? */ int found = 0; skDLLAssignIter(&iter, open_file_list); while (skDLLIterForward(&iter, (void **)&inode) == 0) { if (st.st_ino == *inode) { WARNINGMSG(("Multiple rwsenders attempting" " to send file '%s'"), name); found = 1; break; } } if (!found) { WARNINGMSG(("Filename already exists (from a" " previous run?). Removing '%s'"), destpath); if (unlink(destpath) == 0) { goto reopen; } WARNINGMSG("Failed to unlink '%s': %s", destpath, strerror(errno)); /* treat file as a duplicate */ } } /* else file is a duplicate */ st.st_ino = 0; destpath[0] = dotpath[0] = '\0'; sendString(q, channel, EXTERNAL, SEND_CONN_DUPLICATE(sndr), LOG_WARNING, "Filename already exists (from %s)", sndr->ident); state = FILE_INFO_ERROR_STATE(sndr); pthread_mutex_unlock(&open_file_mutex); break; } /* else, successfully opened placeholder file */ if (fstat(fd, &st) == -1) { CRITMSG("Could not fstat newly created file '%s': %s", destpath, strerror(errno)); st.st_ino = 0; thread_exit = 1; pthread_mutex_unlock(&open_file_mutex); break; } if (skDLListPushTail(open_file_list, &st.st_ino)) { CRITMSG("Unable to grow open file list"); st.st_ino = 0; thread_exit = 1; pthread_mutex_unlock(&open_file_mutex); break; } pthread_mutex_unlock(&open_file_mutex); DEBUGMSG("Created '%s'", destpath); rv = close(fd); fd = -1; if (rv == -1) { CRITMSG("Could not close file '%s': %s", destpath, strerror(errno)); thread_exit = 1; break; } /* Create the dotfile */ rv = snprintf(dotpath, sizeof(dotpath), "%s/%s", destination_dir, dotname); reopen2: fd = open(dotpath, O_RDWR | O_CREAT | O_EXCL, mode); if (fd == -1) { int saveerrno = errno; if (errno == EEXIST) { WARNINGMSG("Filename already exists. Removing '%s'", dotpath); if (unlink(dotpath) == 0) { goto reopen2; } WARNINGMSG("Failed to unlink '%s': %s", dotpath, strerror(errno)); } CRITMSG("Could not create '%s': %s", dotpath, strerror(saveerrno)); thread_exit = 1; dotpath[0] = '\0'; break; } DEBUGMSG("Created '%s'", dotpath); /* Allocate space on disk */ offrv = lseek(fd, size - 1, SEEK_SET); if (offrv == -1) { CRITMSG("Could not allocate disk space for '%s': %s", dotpath, strerror(errno)); thread_exit = 1; break; } rv = write(fd, "", 1); if (rv == -1) { CRITMSG("Could not allocate disk space for '%s': %s", dotpath, strerror(errno)); thread_exit = 1; break; } /* Map space */ map = (uint8_t *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if ((void *)map == MAP_FAILED) { CRITMSG("Could not map '%s': %s", dotpath, strerror(errno)); thread_exit = 1; break; } rv = close(fd); fd = -1; if (rv == -1) { CRITMSG("Could not close file '%s': %s", dotpath, strerror(errno)); thread_exit = 1; break; } GOT_DISK_SPACE(pa_size); pa_size = 0; state = File_info_ack; } break; case File_info_ack: DEBUG_PRINT1("Sending CONN_NEW_FILE_READY"); proto_err = skMsgQueueSendMessage(q, channel, CONN_NEW_FILE_READY, NULL, 0); state = Send_file; break; case Send_file: /* Get the content of the file and write into the dot file */ { block_info_t *block; uint64_t offset; uint32_t len; if (skMsgType(msg) != CONN_FILE_BLOCK) { if ((proto_err = checkMsg(msg, q, CONN_FILE_COMPLETE))) { break; } DEBUG_PRINT1("Received CONN_FILE_COMPLETE"); state = Complete_ack; break; } block = (block_info_t *)skMsgMessage(msg); len = skMsgLength(msg) - offsetof(block_info_t, block); offset = (uint64_t)ntohl(block->high_offset) << 32 | ntohl(block->low_offset); DEBUG_CONTENT_PRINT("Receiving offset=%" PRIu64 " len=%" PRIu32, offset, len); if (offset + len > size) { sendString(q, channel, EXTERNAL, CONN_DISCONNECT, LOG_WARNING, ("Illegal block (offset/size %" PRIu64 "/%" PRIu32 ")"), offset, len); state = Error; break; } memcpy(map + offset, block->block, len); } break; case Complete_ack: /* Un-mmap() the file, create any duplicate files, and * move the dotfile over the placeholder file */ rv = munmap(map, size); map = NULL; if (rv == -1) { CRITMSG("Could not unmap file '%s': %s", dotpath, strerror(errno)); thread_exit = 1; break; } /* Handle duplicate-destinations. Any errors here are * simply logged and processing continues. */ skDLLAssignIter(&iter, duplicate_dirs); while (skDLLIterForward(&iter, (void **)&duplicate_dir) == 0) { char path[sizeof(destpath)]; snprintf(path, sizeof(path), "%s/%s", duplicate_dir, name); if (unique_duplicates) { rv = skCopyFile(dotpath, path); if (rv != 0) { WARNINGMSG("Could not copy '%s' to '%s': %s", dotpath, path, strerror(rv)); } } else { DEBUGMSG("Linking '%s' as '%s'", dotpath, path); rv = link(dotpath, path); if (EXDEV == errno) { DEBUGMSG("Link failed EXDEV; copying '%s' to '%s'", dotpath, path); rv = skCopyFile(dotpath, path); if (rv != 0) { WARNINGMSG("Could not copy '%s' to '%s': %s", dotpath, path, strerror(rv)); } } else if (rv != 0) { WARNINGMSG("Could not link '%s' as '%s': %s", dotpath, path, strerror(errno)); } } } DEBUGMSG("Renaming '%s' to '%s'", dotpath, destpath); rv = rename(dotpath, destpath); if (rv != 0) { CRITMSG("Failed rename of '%s' to '%s': %s", dotpath, destpath, strerror(errno)); thread_exit = 1; break; } /* remove the file from the open_file_list */ pthread_mutex_lock(&open_file_mutex); skDLLAssignIter(&iter, open_file_list); while (skDLLIterForward(&iter, (void **)&inode) == 0) { if (st.st_ino == *inode) { skDLLIterDel(&iter); break; } } st.st_ino = 0; pthread_mutex_unlock(&open_file_mutex); DEBUG_PRINT1("Sending CONN_FILE_COMPLETE"); proto_err = skMsgQueueSendMessage(q, channel, CONN_FILE_COMPLETE, NULL, 0); if (proto_err == 0) { /* Run the post command on the file */ if (post_command) { runPostCommand(post_command, destpath, sndr->ident); } destpath[0] = '\0'; INFOMSG("Finished receiving from %s: '%s'", sndr->ident, name); free(dotname); dotname = NULL; } destpath[0] = dotpath[0] = '\0'; transferred_file = 1; state = File_info; break; case Error: break; } if (msg != NULL) { skMsgDestroy(msg); } } if (fd != -1) { close(fd); } if (map != NULL) { munmap(map, size); } if (dotname != NULL) { free(dotname); } if (dotpath[0] != '\0') { DEBUGMSG("Removing '%s'", dotpath); unlink(dotpath); } if (destpath[0] != '\0') { DEBUGMSG("Removing '%s'", destpath); unlink(destpath); } if (st.st_ino != 0) { skDLLAssignIter(&iter, open_file_list); while (skDLLIterForward(&iter, (void **)&inode) == 0) { if (st.st_ino == *inode) { skDLLIterDel(&iter); break; } } } if (pa_size) { GOT_DISK_SPACE(pa_size); } if (thread_exit) { return -1; } return transferred_file; }
/* * checkDiskSpace(size); * * Verify that we haven't reached the limits of the file system * usage specified by the command line parameters after the * creation of a file of given size (in bytes). * * If we're out of space, return -1. Else, add 'size' to the * 'pre_alloc_size' global variable and return 0. */ static int checkDiskSpace( uint64_t size) { struct statvfs vfs; int64_t free_space, total, newfree; int rv; double percent_used; DEBUGMSG(("Checking for %" PRIu64 " bytes of free space"), size); rv = statvfs(destination_dir, &vfs); if (rv != 0) { CRITMSG("Could not statvfs '%s'", destination_dir); return -1; } /* free bytes is fundamental block size multiplied by the * available (non-privileged) blocks. */ free_space = ((int64_t)vfs.f_frsize * (int64_t)vfs.f_bavail); /* to compute the total (non-privileged) blocks, subtract the * available blocks from the free (privileged) blocks to get * the count of privileged-only blocks, subtract that from the * total blocks, and multiply the result by the block size. */ total = ((int64_t)vfs.f_frsize * ((int64_t)vfs.f_blocks - ((int64_t)vfs.f_bfree - (int64_t)vfs.f_bavail))); pthread_mutex_lock(&pre_alloc_size_mutex); newfree = free_space - pre_alloc_size - size;; percent_used = ((double)(total - newfree) / ((double)total / 100.0)); /* SENDRCV_DEBUG is defined in libsendrcv.h */ #if (SENDRCV_DEBUG) & DEBUG_RWRECEIVER_DISKFREE DEBUGMSG(("frsize: %" PRIu32 "; blocks: %" PRIu64 "; bfree: %" PRIu64 "; bavail: %" PRIu64 "; total: %" PRId64 "; free_space: %" PRId64 "; pre-alloc: %" PRIu64 "; newfree: %" PRId64), (uint32_t)vfs.f_frsize, (uint64_t)vfs.f_blocks, (uint64_t)vfs.f_bfree, (uint64_t)vfs.f_bavail, total, free_space, pre_alloc_size, newfree); #endif if (newfree < freespace_minimum) { CRITMSG(("Free disk space limit overrun: " "free=%" PRId64 " < min=%" PRId64 " (used %.4f%%)"), newfree, freespace_minimum, percent_used); pthread_mutex_unlock(&pre_alloc_size_mutex); return -1; } if (percent_used > space_maximum_percent) { CRITMSG(("Free disk space limit overrun: " "used=%.4f%% > max=%.4f%% (free %" PRId64 " bytes)"), percent_used, space_maximum_percent, newfree); pthread_mutex_unlock(&pre_alloc_size_mutex); return -1; } DEBUGMSG(("Free space available after file of size %" PRIu64 " would be %" PRId64 " bytes (%.4f%%)"), size, newfree, percent_used); pre_alloc_size += size; pthread_mutex_unlock(&pre_alloc_size_mutex); return 0; }