static feedtypet whatami(const char *av0) { feedtypet type; #define SEP '/' /* separates components of path */ /* strip off leading path */ if ((progname = strrchr(av0, SEP)) == NULL) progname = av0; else progname++; type = atofeedtypet(progname); if(type == NONE) type = WMO; /* default for wmo ingestd */ setFeedDefaults(type); return type; }
int main( int ac, char *av[] ) { const char* const pqfname = getQueuePath(); const char* const progname = ubasename(av[0]); int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, /* all files inserted successfully */ exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_dup = 4, /* input-file already in product-queue */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; #if !USE_MMAP pqeIndex = PQE_NONE; #endif { extern int optind; extern int opterr; extern char *optarg; int ch; (void) openulog(progname, LOG_NOTIME, LOG_LDM, "-"); (void) setulogmask(LOG_UPTO(LOG_NOTICE)); opterr = 0; /* Suppress getopt(3) error messages */ while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': (void) setulogmask(getulogmask() | LOG_MASK(LOG_INFO)); break; case 'x': (void) setulogmask(getulogmask() | LOG_MASK(LOG_DEBUG)); break; case 'l': openulog(progname, ulog_get_options(), LOG_LDM, optarg); break; case 'q': setQueuePath(optarg); break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case ':': { LOG_ADD1("Option \"-%c\" requires an operand", optopt); usage(progname); } /* no break */ default: LOG_ADD1("Unknown option: \"%c\"", optopt); usage(progname); /* no break */ } ac -= optind; av += optind ; if(ac < 1) usage(progname); } /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { serror("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } #if USE_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == NULL) { serror("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case PQUEUE_BIG: uerror("Product too big for queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_infile; break; case ENOMEM: uerror("queue full?"); exitCode = exit_system; break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else // USE_MMAP above; !USE_MMAP below status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("xx_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } pqeIndex = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex); if(status != ENOERR) { serror("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { serror("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, pqeIndex); pqeIndex = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into `pqeIndex` region */ if (status != ENOERR) { (void)pqe_discard(pq, pqeIndex); pqeIndex = PQE_NONE; } } /* `pqeIndex` region allocated */ #endif (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
/* * Returns: * 0 Success * SYSTEM_ERROR O/S failure. "log_add()" called. * LP_TIMEDOUT The RPC call timed-out. "log_add()" called. * LP_RPC_ERROR RPC error. "log_add()" called. * LP_LDM_ERROR LDM error. "log_add()" called. */ int main( int ac, char* av[]) { char myname[_POSIX_HOST_NAME_MAX]; char* progname = av[0]; prod_class_t clss; prod_spec spec; int seq_start = 0; int status; ErrorObj* error; unsigned remotePort = LDM_PORT; /* * Set up error logging */ (void)log_init(progname); remote = "localhost"; (void)set_timestamp(&clss.from); clss.to = TS_ENDT; clss.psa.psa_len = 1; clss.psa.psa_val = &spec; spec.feedtype = DEFAULT_FEEDTYPE; spec.pattern = ".*"; { extern int optind; extern char *optarg; int ch; while ((ch = getopt(ac, av, "vxl:h:f:P:s:")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': (void)log_set_destination(optarg); break; case 'h': remote = optarg; break; case 'f': spec.feedtype = atofeedtypet(optarg); if(spec.feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'P': { char* suffix = ""; long port; errno = 0; port = strtol(optarg, &suffix, 0); if (0 != errno || 0 != *suffix || 0 >= port || 0xffff < port) { (void)fprintf(stderr, "%s: invalid port %s\n", av[0], optarg); usage(av[0]); } remotePort = (unsigned)port; break; } case 's': seq_start = atoi(optarg); break; case '?': usage(progname); break; } ac -= optind; av += optind; if(ac < 1) usage(progname); } /* * Register the exit handler */ if(atexit(cleanup) != 0) { log_syserr_q("atexit"); exit(SYSTEM_ERROR); } /* * Set up signal handlers */ set_sigactions(); (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; (void)exitIfDone(INTERRUPTED); /* * Connect to the LDM. */ status = lp_new(remote, &ldmProxy); if (0 != status) { log_flush_error(); status = (LP_SYSTEM == status) ? SYSTEM_ERROR : CONNECTION_ABORTED; } else { log_debug("version %u", lp_version(ldmProxy)); status = ldmsend(ldmProxy, &clss, myname, seq_start, ac, av); if (0 != status) log_flush_error(); lp_free(ldmProxy); ldmProxy = NULL; } /* "ldmProxy" allocated */ return status; }
int main(int ac, char *av[]) { int logfd; int width; int ready; unsigned long idle; fd_set readfds; fd_set exceptfds; struct timeval timeo; const char* const progname = basename(av[0]); unsigned long maxProductSize = DEFAULT_MAX_PRODUCT_SIZE; /* * Setup default logging before anything else. */ (void)log_init(progname); feedtype = whatami(av[0]); { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 0; /* stops getopt() from printing to stderr */ usePil = 1; useNex = 1; while ((ch = getopt(ac, av, ":vxcni5Nl:b:p:P:T:q:r:f:s:")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'c': chkflag = CHK_CHECK; break; case 'n': chkflag = CHK_DONT; break; case 'i': usePil = 0; break; case 'N': useNex = 0; break; case '5': skipLeadingCtlString = 0; break; case 'l': { (void)log_set_destination(optarg); break; } case 'b': baud = optarg; break; case 'p': parity = optarg; break; #if NET case 'P': *((int *)&server_port) = atoi(optarg); /* cast away const */ if(server_port <= 0 || server_port > 65536) { log_error("Invalid server port: \"%s\"", optarg); usage(progname); } break; case 'T': reset_secs = atoi(optarg); if(reset_secs < 0) { log_add("Invalid timeout: \"%s\"", optarg); usage(progname); } break; #endif /* NET */ case 's': { unsigned long size; int nbytes; if (sscanf(optarg, "%lu %n", &size, &nbytes) != 1 || optarg[nbytes] != 0 || 1 > size) { log_error("Invalid maximum data-product size: \"%s\"", optarg); usage(progname); } maxProductSize = size; break; } case 'q': setQueuePath(optarg); break; case 'r': rawfname = optarg; break; case 'f': { feedtypet type; type = atofeedtypet(optarg); if(type != NONE) { feedtype = type; if(!parity && !baud) setFeedDefaults(type); } } break; case '?': { log_add("Unknown option: \"%c\"", optopt); usage(progname); break; } case ':': /*FALLTHROUGH*/ default: log_add("Missing argument for option: \"%c\"", optopt); usage(progname); break; } /* last arg, feedfname, is required */ if(ac - optind != 1) { log_add("Wrong number of operands: %d", ac - optind); usage(progname); } (void)strncat(feedfname, av[optind], sizeof(feedfname)-6); } pqpath = getQueuePath(); log_notice("Starting Up"); log_debug(PACKAGE_VERSION); /* * register exit handler */ if(atexit(cleanup) != 0) { log_syserr("atexit"); return 1; } /* * set up signal handlers */ set_sigactions(); /* * open the product queue, unless we were invoked as "feedtest" */ if(strcmp(progname, "feedtest") != 0) { if((ready = pq_open(pqpath, PQ_DEFAULT, &pq))) { if (PQ_CORRUPT == ready) { log_error("The product-queue \"%s\" is inconsistent\n", pqpath); } else { log_error("pq_open: \"%s\" failed: %s", pqpath, strerror(ready)); } return 1; } } /* * who am i, anyway */ (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; /* * open the feed */ if(!(*feedfname == '-' && feedfname[1] == 0) && logfd != 0) (void) close(0); if(open_feed(feedfname, &ifd, maxProductSize) != ENOERR) return 1; if(usePil == 1) { if ((feedtype & DDS)||(feedtype & PPS)||(feedtype & IDS)|| (feedtype & HRS)) { usePil = 1; log_info("Creating AFOS-like pil tags\0"); } else { usePil = 0; } } if (feedtype & HDS) { if(chkflag == CHK_CHECK || (isatty(ifd) && chkflag != CHK_DONT)) setTheScanner(scan_wmo_binary_crc); else setTheScanner(scan_wmo_binary); } else if (feedtype == ( DDPLUS | IDS ) ) { /* this is the combined NOAAPORT fos-alike. We know these have the 4 byte start and end sequences. Using the binary scanner ensures that we don't stop on an arbitray embedded CTRL-C */ log_notice("Note: Using the wmo_binary scanner for SDI ingest\0"); setTheScanner (scan_wmo_binary); } else if (feedtype & (NMC2 | NMC3)) { setTheScanner(scan_wmo_binary); } else if (feedtype == AFOS) { prod_stats = afos_stats; setTheScanner(scan_afos); } else if (feedtype == FAA604) { prod_stats = faa604_stats; if(chkflag == CHK_CHECK || (isatty(ifd) && chkflag != CHK_DONT && parity != NULL && *parity != 'n') ) { setTheScanner(scan_faa604_parity); } else { setTheScanner(scan_faa604); } } else { if(chkflag == CHK_CHECK || (isatty(ifd) && chkflag != CHK_DONT && parity != NULL && *parity != 'n') ) { setTheScanner(scan_wmo_parity); } else { setTheScanner(scan_wmo); } } /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { log_syserr("new_md5_CTX failed"); return 1; } /* * Main Loop */ idle = 0; while(exitIfDone(0)) { #if NET if (INPUT_IS_SOCKET) { if (port_error) { /* * lost connection => close */ if (ifd >= 0) { if(feed_close) (*feed_close)(ifd); ifd = -1; } port_error = 0; sleep (2); /* allow things to settle down */ continue; } } #endif if(stats_req) { log_notice("Statistics Request"); if(pq != NULL) { off_t highwater = 0; size_t maxregions = 0; (void) pq_highwater(pq, &highwater, &maxregions); log_notice(" Queue usage (bytes):%8ld", (long)highwater); log_notice(" (nregions):%8ld", (long)maxregions); } log_notice(" Idle: %8lu seconds", idle); #if NET if (INPUT_IS_SOCKET) { log_notice(" Timeout: %8d", reset_secs); } #endif log_notice("%21s: %s", "Status", (ifd < 0) ? "Not connected or input not open." : "Connected."); (*prod_stats)(); (*feed_stats)(); stats_req = 0; } #if NET if (INPUT_IS_SOCKET) { if (ifd < 0) { /* Attempt reconnect */ static int retries = 0; if (retries > MAX_RETRIES) { log_error ("maximum retry attempts %d, aborting", MAX_RETRIES); done = !0; continue; } /* Try to reopen on tcp read errors */ log_notice("Trying to re-open connection on port %d", server_port); ++retries; if(open_feed(feedfname, &ifd, maxProductSize) != ENOERR) { log_notice ("sleeping %d seconds before retry %d", retries * RETRY_DELAY, retries+1); sleep (retries * RETRY_DELAY); continue; } retries = 0; } } #endif /* NET */ timeo.tv_sec = 3; timeo.tv_usec = 0; FD_ZERO(&readfds); FD_ZERO(&exceptfds); FD_SET(ifd, &readfds); FD_SET(ifd, &exceptfds); width = ifd + 1; ready = select(width, &readfds, 0, &exceptfds, &timeo); if(ready < 0 ) { /* handle EINTR as a special case */ if(errno == EINTR) { errno = 0; continue; } log_syserr("select"); return 1; } /* else */ #if 0 if (FD_ISSET(ifd, &exceptfds)) { log_error("Exception on input fd %d, select returned %d", ifd, ready); } #endif if(ready > 0) { /* do some work */ if(FD_ISSET(ifd, &readfds) || FD_ISSET(ifd, &exceptfds)) { idle = 0; if(feedTheXbuf(ifd) != ENOERR) { #if NET if (INPUT_IS_SOCKET) { port_error = !0; continue; } /* else */ #endif /* NET */ done = !0; } FD_CLR(ifd, &readfds); FD_CLR(ifd, &exceptfds); } else { log_error("select returned %d but ifd not set", ready); idle += timeo.tv_sec; } } else /* ready == 0 */ { idle += timeo.tv_sec; #if NET if (INPUT_IS_SOCKET) { /* VOODOO * This is necessary to stimulate * 'Connection reset by peer' * when the Portmaster goes down and comes * back up. */ static char zed[1] = {0}; if(write(ifd, zed, sizeof(zed)) < 0) { port_error = !0; continue; } } #endif } #if NET if (INPUT_IS_SOCKET) { if ((reset_secs > 0) && (idle >= reset_secs)) { log_notice("Idle for %ld seconds, reconnecting", idle); /* force reconnect */ port_error = !0; idle = 0; continue; } } #endif /* NET */ (void) scanTheXbuf(); } return 0; }
int main( int ac, char *av[] ) { char *progname = av[0]; char *logfname; int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; logfname = "-"; /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv("LDMPQFNAME"); if(ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) | LOG_MASK(LOG_NOTICE)); opterr = 1; while ((ch = getopt(ac, av, "ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': logmask |= LOG_MASK(LOG_INFO); break; case 'x': logmask |= LOG_MASK(LOG_DEBUG); break; case 'l': logfname = optarg; break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case '?': usage(progname); break; } ac -= optind; av += optind ; if(ac < 1) usage(progname); (void) setulogmask(logmask); } /* * Set up error logging */ (void) openulog(ubasename(progname), LOG_NOTIME, LOG_LDM, logfname); /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strcpy(myname, ghostname()); /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { serror("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; #ifndef NO_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == NULL) { serror("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else /*!NO_MMAP*/ status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("fd_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } index = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &index); if(status != ENOERR) { serror("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { serror("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, index); index = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into "index" region */ if (status != ENOERR) { (void)pqe_discard(pq, index); index = PQE_NONE; } } /* "index" region allocated */ #endif /*!NO_MMAP*/ (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
int main (int ac, char *av[]) { char *progname = av[0]; int status; int seq_start = 0; stat_info *sinfo, *shead = NULL, *slast = NULL; int statusoff=0; /* * Set up error logging */ if (log_init(progname)) { log_syserr("Couldn't initialize logging module"); exit(1); } const char* pqfname = getQueuePath(); /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv ("LDMPQFNAME"); if (ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 1; while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': if (log_set_destination(optarg)) { log_syserr("Couldn't set logging destination to \"%s\"", optarg); exit(1); } break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi (optarg); break; case 'f': feedtype = atofeedtypet (optarg); if (feedtype == NONE) { fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg); usage (progname); } break; case 'S': statusoff=1; break; case '?': usage (progname); break; } setQueuePath(pqfname); ac -= optind; av += optind; if (ac < 1) usage (progname); } /* * register exit handler */ if (atexit (cleanup) != 0) { log_syserr ("atexit"); exit (1); } /* * set up signal handlers */ set_sigactions (); /* * who am i, anyway */ (void) strcpy (myname, ghostname ()); /* * open the product queue */ if ((status = pq_open (pqfname, PQ_DEFAULT, &pq))) { if (status > 0) { log_add_syserr("\"%s\" failed", pqfname); log_flush_error(); } else { log_error_q("\"%s\" failed: %s", pqfname, "Internal error"); } exit (2); } { char *filename; int fd; struct stat statb; product prod; unsigned char *prodmmap; MD5_CTX *md5ctxp = NULL; int gversion; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX (); if (md5ctxp == NULL) { log_syserr ("new_md5_CTX failed"); exit (6); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; prod.info.seqno = seq_start; /* * Open the file to be inserted and process */ while (ac > 0) { long insert_sum = 0; long sinfo_cnt = 0; long stat_size = 0; filename = *av; av++; ac--; log_notice_q ("open and memorymap %s\0", filename); fd = open (filename, O_RDONLY, 0); if (fd == -1) { log_syserr ("open: %s", filename); continue; } if (fstat (fd, &statb) == -1) { log_syserr ("fstat: %s", filename); (void) close (fd); continue; } if ((prodmmap = (unsigned char *) mmap (0, statb.st_size, PROT_READ, MAP_PRIVATE, fd, 0)) == MAP_FAILED) { log_syserr ("allocation failed"); } else { int GRIBDONE = 0; off_t griboff = 0; size_t griblen = 0; log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size); while (!GRIBDONE) { log_debug("griboff %d\0", (int) griboff); /* get offset of next grib product */ status = get_grib_info (prodmmap, statb.st_size, &griboff, &griblen, &gversion); switch (status) { case 0: prod.data = prodmmap + griboff; prod.info.sz = griblen; /* * revised MD5 calculation...using filename * to allow duplicate products in different files. */ MD5Init (md5ctxp); MD5Update (md5ctxp, (void *)filename, strlen(filename)); /*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/ if ( prod.info.sz > 10000 ) MD5Update (md5ctxp, (void *)prod.data, 10000); else MD5Update (md5ctxp, (void *)prod.data, prod.info.sz); MD5Final (prod.info.signature, md5ctxp); /*if (mm_md5 (md5ctxp, prod.data, prod.info.sz, prod.info.signature) != 0) { log_error_q ("could not compute MD5\0"); } else { */ prod.info.ident = (char *) malloc (KEYSIZE + 1); get_gribname (gversion, prod.data, prod.info.sz, filename, prod.info.seqno, prod.info.ident); /* * Do the deed */ status = set_timestamp (&prod.info.arrival); if (status != ENOERR) { log_add_syserr("could not set timestamp"); log_flush_error(); } /* * Insert the product */ status = pq_insert (pq, &prod); log_info_q ("%d %s\0", status, prod.info.ident); if ( status == ENOERR ) insert_sum += prod.info.sz; if (! statusoff ) { /* * Log this status */ sinfo_cnt++; sinfo = (stat_info *)malloc(sizeof(stat_info)); sinfo->insertstatus = status; sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1); strcpy(sinfo->prodname, prod.info.ident); sinfo->seqno = prod.info.seqno; sinfo->prodsz = prod.info.sz; sinfo->next = NULL; stat_size += strlen(sinfo->prodname); if(shead == NULL) { shead = sinfo; slast = sinfo; } else { slast->next = sinfo; slast = sinfo; } } /*}*/ griboff += griblen; prod.info.seqno++; break; case -1: GRIBDONE = 1; break; case -2: log_error_q ("truncated grib file at: %d", prod.info.seqno); GRIBDONE = 1; break; case -7: log_error_q ("End sequence 7777 not found where expected: %d", prod.info.seqno); griboff += griblen; log_error_q("resume looking at %d\0",griboff); break; default: log_error_q ("unknown error %d\0", status); griboff += griblen; if (griboff >= statb.st_size) GRIBDONE = 1; break; } if (griboff >= statb.st_size) GRIBDONE = 1; } log_notice_q ("munmap\0"); (void) munmap ((void *)prodmmap, statb.st_size); if ( stat_size != 0 ) /* * Add a status message to product queue */ { char *statusmess; log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt); statusmess = calloc((30 * sinfo_cnt) + stat_size + strlen(filename) + 128, sizeof(char)); if(statusmess == NULL) { log_syserr("could not malloc status message %ld\0", stat_size); } else { char tmpprod[512]; sinfo = shead; slast = NULL; status = set_timestamp(&prod.info.arrival); /* ctime ends with \n\0" */ sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n", filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec), insert_sum,(long)statb.st_size); while(sinfo != NULL) { sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus, sinfo->seqno,sinfo->prodsz,sinfo->prodname); strcat(statusmess,tmpprod); slast = sinfo; sinfo = sinfo->next; free(slast->prodname); free(slast); } shead = NULL; sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno); prod.info.ident = tmpprod; prod.data = statusmess; prod.info.sz = strlen(statusmess); status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); status = set_timestamp(&prod.info.arrival); status = pq_insert(pq, &prod); if(log_is_enabled_info) log_info_q("%s", s_prod_info(NULL, 0, &prod.info, log_is_enabled_debug)) ; free(statusmess); prod.info.seqno++; } } } (void) close (fd); } } exit(0); }
int main(int ac, char *av[]) { char myname[HOSTNAMESIZE]; char *progname = av[0]; char *logfname; unsigned version; prod_class_t clss; prod_spec spec; int seq_start = 0; int status; ErrorObj* error; unsigned remotePort = LDM_PORT; logfname = "-"; remote = "localhost"; (void)set_timestamp(&clss.from); clss.to = TS_ENDT; clss.psa.psa_len = 1; clss.psa.psa_val = &spec; spec.feedtype = DEFAULT_FEEDTYPE; spec.pattern = ".*"; { extern int optind; extern char *optarg; int ch; int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) | LOG_MASK(LOG_NOTICE)); while ((ch = getopt(ac, av, "vxl:h:f:P:s:")) != EOF) switch (ch) { case 'v': logmask |= LOG_MASK(LOG_INFO); break; case 'x': logmask |= LOG_MASK(LOG_DEBUG); break; case 'l': logfname = optarg; break; case 'h': remote = optarg; break; case 'f': spec.feedtype = atofeedtypet(optarg); if(spec.feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'P': { char* suffix = ""; long port; errno = 0; port = strtol(optarg, &suffix, 0); if (0 != errno || 0 != *suffix || 0 >= port || 0xffff < port) { (void)fprintf(stderr, "%s: invalid port %s\n", av[0], optarg); usage(av[0]); } remotePort = (unsigned)port; break; } case 's': seq_start = atoi(optarg); break; case '?': usage(progname); break; } ac -= optind; av += optind; if(ac < 1) usage(progname); (void) setulogmask(logmask); } /* * Set up error logging */ (void) openulog(ubasename(progname), LOG_NOTIME, LOG_LDM, logfname); /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(1); } /* * set up signal handlers */ set_sigactions(); (void) strcpy(myname, ghostname()); /* * Contact the server. */ error = ldm_clnttcp_create_vers(remote, remotePort, SIX, &clnt, NULL, NULL); (void)exitIfDone(1); if (!error) { version = SIX; hiya = my_hiya_6; send_product = send_product_6; nullproc = nullproc_6; } else if (LDM_CLNT_BAD_VERSION == err_code(error)) { err_free(error); error = ldm_clnttcp_create_vers(remote, remotePort, FIVE, &clnt, NULL, NULL); (void)exitIfDone(1); if (!error) { version = FIVE; hiya = my_hiya_5; send_product = send_product_5; nullproc = NULL; } } if (error) { err_log(error, ERR_FAILURE); err_free(error); status = 1; } else { udebug("version %u", version); status = ldmsend(clnt, &clss, myname, seq_start, ac, av); } return status != 0; }