/** * Returns a new product-maker. * * This function is thread-safe. * * @retval 0 Success. * @retval 1 Usage failure. \c log_start() called. * @retval 2 O/S failure. \c log_start() called. */ int pmNew( Fifo* const fifo, /**< [in] Pointer to FIFO from * which to get data */ LdmProductQueue* const lpq, /**< [in] LDM product-queue into * which to put data-products */ ProductMaker** const productMaker) /**< [out] Pointer to pointer to * returned product-maker */ { int status = 2; /* default failure */ ProductMaker* w = (ProductMaker*)malloc(sizeof(ProductMaker)); if (NULL == w) { LOG_SERROR0("Couldn't allocate new product-maker"); } else { MD5_CTX* md5ctxp = new_MD5_CTX(); if (NULL == md5ctxp) { LOG_SERROR0("Couldn't allocate MD5 object"); } else { if ((status = pthread_mutex_init(&w->mutex, NULL)) != 0) { LOG_ERRNUM0(status, "Couldn't initialize product-maker mutex"); status = 2; } else { w->fifo = fifo; w->ldmProdQueue = lpq; w->npackets = 0; w->nmissed = 0; w->nprods = 0; w->md5ctxp = md5ctxp; w->status = 0; *productMaker = w; } } } return status; }
int main( int ac, char *av[] ) { const char* const pqfname = getQueuePath(); const char* const progname = ubasename(av[0]); int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, /* all files inserted successfully */ exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_dup = 4, /* input-file already in product-queue */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; #if !USE_MMAP pqeIndex = PQE_NONE; #endif { extern int optind; extern int opterr; extern char *optarg; int ch; (void) openulog(progname, LOG_NOTIME, LOG_LDM, "-"); (void) setulogmask(LOG_UPTO(LOG_NOTICE)); opterr = 0; /* Suppress getopt(3) error messages */ while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': (void) setulogmask(getulogmask() | LOG_MASK(LOG_INFO)); break; case 'x': (void) setulogmask(getulogmask() | LOG_MASK(LOG_DEBUG)); break; case 'l': openulog(progname, ulog_get_options(), LOG_LDM, optarg); break; case 'q': setQueuePath(optarg); break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case ':': { LOG_ADD1("Option \"-%c\" requires an operand", optopt); usage(progname); } /* no break */ default: LOG_ADD1("Unknown option: \"%c\"", optopt); usage(progname); /* no break */ } ac -= optind; av += optind ; if(ac < 1) usage(progname); } /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { serror("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } #if USE_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == NULL) { serror("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case PQUEUE_BIG: uerror("Product too big for queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_infile; break; case ENOMEM: uerror("queue full?"); exitCode = exit_system; break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else // USE_MMAP above; !USE_MMAP below status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("xx_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } pqeIndex = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex); if(status != ENOERR) { serror("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { serror("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, pqeIndex); pqeIndex = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into `pqeIndex` region */ if (status != ENOERR) { (void)pqe_discard(pq, pqeIndex); pqeIndex = PQE_NONE; } } /* `pqeIndex` region allocated */ #endif (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
/** * Reads NOAAPORT data from a shared-memory FIFO or a file, creates LDM * data-products, and inserts the data-products into an LDM product-queue. * * Usage: * * readnoaaport [-nvx] [-q <em>queue</em>] [-u <em>n</em>] [-m mcastAddr] [path]\n * * Where: * <dl> * <dt>-l <em>log</em></dt> * <dd>Log to \e log. if \e log is "-", then logging occurs to the * standard error stream; otherwise, \e log is the pathname of a file to * which logging will occur. If not specified, then log messages will go * to the system logging daemon. </dd> * * <dt>-m <em>mcastAddr</em></dt> * <dd>Use the shared-memory FIFO associated with the UDP * multicast address \e mcastAddr.</dd> * * <dt>-n</dt> * <dd>Log messages of level NOTICE and higher priority.</dd> * * <dt>-q <em>queue</em></dt> * <dd>Use \e queue as the pathname of the LDM product-queue. The default * is to use the default LDM pathname of the product-queue.</dd> * * <dt>-u <em>n</em></dt> * <dd>If logging is to the system logging daemon, then use facility * <b>local</b><em>n</em>. The default is to use the LDM facility. * * <dt>-v</dt> * <dd>Log messages of level INFO and higher priority. Each data-product * will generate a log message.</dd> * * <dt>-x</dt> * <dd>Log messages of level DEBUG and higher priority.</dd> * * <dt><em>path</em></dt> * <dd>Pathname of the file from which to read data. The default is to use * a shared-memory FIFO.</dd> * </dl> * * @retval 0 if successful. * @retval 1 if an error occurred. At least one error-message is logged. */ int main( const int argc, char* const argv[]) { #ifdef HAVE_GET_QUEUE_PATH const char* pqfname = getQueuePath(); #else const char* pqfname = DEFAULT_QUEUE; #endif int fd; char* prodmmap; char* memheap = NULL; size_t heapsize; size_t heapcount; unsigned char b1; int cnt, dataoff, datalen, deflen; int nscan; long IOFF; int NWSTG, GOES, PNGINIT = 0, PROD_COMPRESSED; long last_sbn_seqno = (-1); char PROD_NAME[1024]; int status; prodstore prod; sbn_struct* sbn; pdh_struct* pdh; psh_struct* psh; ccb_struct* ccb; pdb_struct* pdb; datastore* pfrag; extern int optind; extern int opterr; extern char* optarg; int ch; int logmask = LOG_MASK(LOG_ERR); const char* logfname = NULL; /* use system logging daemon */ unsigned logOptions = LOG_CONS | LOG_PID; unsigned logFacility = LOG_LDM; /* use default LDM facility */ const char* const progName = ubasename(argv[0]); MD5_CTX* md5ctxp = NULL; /*unsigned char *compr; long comprLen = 10000 * sizeof (int);*/ int pid_channel = -1; /*compr = (unsigned char *) calloc (comprLen, 1);*/ /* Initialize the logger. */ (void)setulogmask(logmask); (void)openulog(progName, logOptions, logFacility, logfname); opterr = 1; while ((ch = getopt(argc, argv, "nvxl:q:u:m:")) != EOF) { switch (ch) { case 'v': logmask |= LOG_MASK(LOG_INFO); (void)setulogmask(logmask); break; case 'x': logmask |= LOG_MASK(LOG_DEBUG); (void)setulogmask(logmask); break; case 'n': logmask |= LOG_MASK(LOG_NOTICE); (void)setulogmask(logmask); break; case 'l': if (optarg[0] == '-' && optarg[1] != 0) { nplError("logfile \"%s\" ??\n", optarg); usage(argv[0]); } /* else */ logfname = optarg; (void)openulog(progName, logOptions, logFacility, logfname); break; case 'q': pqfname = optarg; break; case 'u': { int i = atoi(optarg); if (0 <= i && 7 >= i) { static int logFacilities[] = {LOG_LOCAL0, LOG_LOCAL1, LOG_LOCAL2, LOG_LOCAL3, LOG_LOCAL4, LOG_LOCAL5, LOG_LOCAL6, LOG_LOCAL7}; logFacility = logFacilities[i]; (void)openulog(progName, logOptions, logFacility, logfname); } break; } case 'm': sscanf(optarg, "%*d.%*d.%*d.%d", &pid_channel); if ((pid_channel < 1) || (pid_channel > MAX_DVBS_PID)) { pid_channel = -1; } else { shm = shmfifo_new(); cnt = 0; while (((status = shmfifo_shm_from_key(shm, s_port[pid_channel - 1])) == -3) && (cnt < 30)) { nplInfo("Trying to get shared-memory FIFO"); cnt++; sleep(1); } if (0 != status) { nplError("Couldn't get shared-memory FIFO. " "Check associated dvbs_multicast(1) process."); shmfifo_free(shm); shm = NULL; } else { nplInfo("Got shared-memory FIFO"); } } break; case '?': usage(argv[0]); break; } } if (argc - optind < 0) usage(argv[0]); nplNotice("Starting Up %s", PACKAGE_VERSION); fd = ((argc - optind) == 0) ? fileno(stdin) : open(argv[optind], O_RDONLY, 0); if ((!shm) && (fd == -1)) { nplError("could not open input file"); exit(0); } /* * Set up signal handlers */ set_sigactions(); /* * Register atexit routine */ if (atexit(cleanup) != 0) { nplSerror("atexit"); exit(-1); } sbn = (sbn_struct*)malloc(sizeof(sbn_struct)); pdh = (pdh_struct*)malloc(sizeof(pdh_struct)); psh = (psh_struct*)malloc(sizeof(psh_struct)); ccb = (ccb_struct*)malloc(sizeof(ccb_struct)); pdb = (pdb_struct*)malloc(sizeof(pdb_struct)); prodmmap = (char*)malloc(10000); if (prodmmap == NULL) { nplError("could not allocate read buffer"); exit(-1); } md5ctxp = new_MD5_CTX(); prod.head = NULL; prod.tail = NULL; if (lpqGet(pqfname, &ldmProdQueue) != 0) { NPL_ADD1("Couldn't open LDM product-queue \"%s\"", pqfname); exit(1); } while (DONE == 0) { /* See if any stats need to be logged */ if (logstats) { logstats = 0; dump_stats(); } /* Look for first byte == 255 and a valid SBN checksum */ if ((status = bufread(fd, prodmmap, 1)) != 0) { if (-3 == status) break; abort(); } if ((b1 = (unsigned char)prodmmap[0]) != 255) { if (ulogIsVerbose()) nplInfo("trying to resync %u", b1); if (ulogIsDebug()) nplDebug("bufread loop"); continue; } if (bufread(fd, prodmmap + 1, 15) != 0) { if (ulogIsDebug()) nplDebug("couldn't read 16 bytes for sbn"); continue; } while ((status = readsbn(prodmmap, sbn)) != 0) { if (ulogIsDebug()) nplDebug("Not SBN start"); IOFF = 1; while ((IOFF < 16) && ((b1 = (unsigned char) prodmmap[IOFF]) != 255)) IOFF++; if (IOFF > 15) { break; } else { for (ch = IOFF; ch < 16; ch++) prodmmap[ch - IOFF] = prodmmap[ch]; if (bufread(fd, prodmmap + 16 - IOFF, IOFF) != 0) { if (ulogIsDebug()) nplDebug("Couldn't read bytes for SBN, resync"); break; } } } if (status != 0) { if (ulogIsDebug()) nplDebug("SBN status continue"); continue; } IOFF = 0; if (bufread(fd, prodmmap + 16, 16) != 0) { if (ulogIsDebug()) nplDebug("error reading Product Definition Header"); continue; } if (ulogIsDebug()) nplDebug("***********************************************"); if (last_sbn_seqno != -1) { if (sbn->seqno != last_sbn_seqno + 1) { nplNotice("Gap in SBN sequence number %ld to %ld [skipped %ld]", last_sbn_seqno, sbn->seqno, sbn->seqno - last_sbn_seqno - 1); if ( sbn->seqno > last_sbn_seqno ) nmissed = nmissed + (unsigned long)(sbn->seqno - last_sbn_seqno - 1); } } last_sbn_seqno = sbn->seqno; if (ulogIsVerbose()) nplInfo("SBN seqnumber %ld", sbn->seqno); if (ulogIsVerbose()) nplInfo("SBN datastream %d command %d", sbn->datastream, sbn->command); if (ulogIsDebug()) nplDebug("SBN version %d length offset %d", sbn->version, sbn->len); if (((sbn->command != 3) && (sbn->command != 5)) || (sbn->version != 1)) { nplError("Unknown sbn command/version %d PUNT", sbn->command); continue; } switch (sbn->datastream) { case 7: /* test */ case 6: /* was reserved...now nwstg2 */ case 5: NWSTG = 1; GOES = 0; break; case 1: case 2: case 4: NWSTG = 0; GOES = 1; break; default: nplError("Unknown NOAAport channel %d PUNT", sbn->datastream); continue; } /* End of SBN version low 4 bits */ if (readpdh(prodmmap + IOFF + sbn->len, pdh) == -1) { nplError("problem with pdh, PUNT"); continue; } if (pdh->len > 16) { bufread(fd, prodmmap + sbn->len + 16, pdh->len - 16); } if (ulogIsDebug()) nplDebug("Product definition header version %d pdhlen %d", pdh->version, pdh->len); if (pdh->version != 1) { nplError("Error: PDH transfer type %u, PUNT", pdh->transtype); continue; } else if (ulogIsDebug()) { nplDebug("PDH transfer type %u", pdh->transtype); } if ((pdh->transtype & 8) > 0) nplError("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 32) > 0) nplError("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 16) > 0) { PROD_COMPRESSED = 1; if (ulogIsDebug()) nplDebug("Product transfer flag compressed %u", pdh->transtype); } else { PROD_COMPRESSED = 0; } if (ulogIsDebug()) nplDebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen, pdh->pshlen); if (ulogIsDebug()) nplDebug("blocks per record %ld records per block %ld\n", pdh->blocks_per_record, pdh->records_per_block); if (ulogIsDebug()) nplDebug("product seqnumber %ld block number %ld data block size " "%ld", pdh->seqno, pdh->dbno, pdh->dbsize); /* Stop here if no psh */ if ((pdh->pshlen == 0) && (pdh->transtype == 0)) { IOFF = IOFF + sbn->len + pdh->len; continue; } if (pdh->pshlen != 0) { if (bufread(fd, prodmmap + sbn->len + pdh->len, pdh->pshlen) != 0) { nplError("problem reading psh"); continue; } else { if (ulogIsDebug()) nplDebug("read psh %d", pdh->pshlen); } /* Timing block */ if (sbn->command == 5) { if (ulogIsDebug()) nplDebug("Timing block recieved %ld %ld\0", psh->olen, pdh->len); /* * Don't step on our psh of a product struct of prod in * progress. */ continue; } if (readpsh(prodmmap + IOFF + sbn->len + pdh->len, psh) == -1) { nplError("problem with readpsh"); continue; } if (psh->olen != pdh->pshlen) { nplError("ERROR in calculation of psh len %ld %ld", psh->olen, pdh->len); continue; } if (ulogIsDebug()) nplDebug("len %ld", psh->olen); if (ulogIsDebug()) nplDebug("product header flag %d, version %d", psh->hflag, psh->version); if (ulogIsDebug()) nplDebug("prodspecific data length %ld", psh->psdl); if (ulogIsDebug()) nplDebug("bytes per record %ld", psh->bytes_per_record); if (ulogIsDebug()) nplDebug("Fragments = %ld category %d ptype %d code %d", psh->frags, psh->pcat, psh->ptype, psh->pcode); if (psh->frags < 0) nplError("check psh->frags %d", psh->frags); if (psh->origrunid != 0) nplError("original runid %d", psh->origrunid); if (ulogIsDebug()) nplDebug("next header offset %ld", psh->nhoff); if (ulogIsDebug()) nplDebug("original seq number %ld", psh->seqno); if (ulogIsDebug()) nplDebug("receive time %ld", psh->rectime); if (ulogIsDebug()) nplDebug("transmit time %ld", psh->transtime); if (ulogIsDebug()) nplDebug("run ID %ld", psh->runid); if (ulogIsDebug()) nplDebug("original run id %ld", psh->origrunid); if (prod.head != NULL) { nplError("OOPS, start of new product [%ld ] with unfinished " "product %ld", pdh->seqno, prod.seqno); ds_free(); prod.head = NULL; prod.tail = NULL; if (PNGINIT != 0) { pngout_end(); PNGINIT = 0; } nplError("Product definition header version %d pdhlen %d", pdh->version, pdh->len); nplError("PDH transfer type %u", pdh->transtype); if ((pdh->transtype & 8) > 0) nplError("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 32) > 0) nplError("Product transfer flag error %u", pdh->transtype); nplError("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen, pdh->pshlen); nplError("blocks per record %ld records per block %ld", pdh->blocks_per_record, pdh->records_per_block); nplError("product seqnumber %ld block number %ld data block " "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize); nplError("product header flag %d", psh->hflag); nplError("prodspecific data length %ld", psh->psdl); nplError("bytes per record %ld", psh->bytes_per_record); nplError("Fragments = %ld category %d", psh->frags, psh->pcat); if (psh->frags < 0) nplError("check psh->frags %d", psh->frags); if (psh->origrunid != 0) nplError("original runid %d", psh->origrunid); nplError("next header offset %ld", psh->nhoff); nplError("original seq number %ld", psh->seqno); nplError("receive time %ld", psh->rectime); nplError("transmit time %ld", psh->transtime); nplError("run ID %ld", psh->runid); nplError("original run id %ld", psh->origrunid); } prod.seqno = pdh->seqno; prod.nfrag = psh->frags; ds_init(prod.nfrag); /* NWSTG CCB = dataoff, WMO = dataoff + 24 */ if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen, pdh->dbsize) != 0) { nplError("problem reading datablock"); continue; } if (sbn->datastream == 4) { if (psh->pcat != 3) { GOES = 0; NWSTG = 1; } } heapcount = 0; MD5Init(md5ctxp); if (GOES == 1) { if (readpdb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen, psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) { nplError("Error reading pdb, punt"); continue; } memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME)); if (ulogIsDebug()) nplDebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len, pdh->pshlen, sbn->len + pdh->len + pdh->pshlen, pdb->len); /* Data starts at first block after pdb */ ccb->len = 0; heapsize = prodalloc(psh->frags, 5152, &memheap); } if (NWSTG == 1) { memset(psh->pname, 0, sizeof(psh->pname)); if (readccb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen, ccb, psh, pdh->dbsize) == -1) nplError("Error reading ccb, using default name"); if (ulogIsDebug()) nplDebug("look at ccb start %d %d", ccb->b1, ccb->len); /* cnt = 0; memset(psh->pname,0,sizeof(psh->pname)); while ((b1 = (unsigned char)prodmmap[ IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len + cnt]) >= 32) { psh->pname[cnt] = prodmmap[ IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len + cnt]; cnt++; } if(cnt > 0) */ if (ulogIsVerbose()) nplInfo("%s", psh->pname); memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME)); heapsize = prodalloc(psh->frags, 4000 + 15, &memheap); /* * We will only compute md5 checksum on the data, 11 FOS * characters at start */ /* * sprintf(memheap,"\001\015\015\012%04d\015\015\012", * ((int)pdh->seqno)%10000); */ sprintf(memheap, "\001\015\015\012%03d\040\015\015\012", ((int) pdh->seqno) % 1000); heapcount += 11; if (psh->metaoff > 0) psh->metaoff = psh->metaoff + 11; } } else { /* If a continuation record...don't let psh->pcat get missed */ if ((sbn->datastream == 4) && (psh->pcat != 3)) { GOES = 0; NWSTG = 1; } ccb->len = 0; if (ulogIsDebug()) nplDebug("continuation record"); if ((pdh->transtype & 4) > 0) { psh->frags = 0; } if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen, pdh->dbsize) != 0) { nplError("problem reading datablock (cont)"); continue; } if (prod.head == NULL) { if (ulogIsVerbose()) nplInfo("found data block before header, " "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno); continue; } } /* Get the data */ dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len; datalen = pdh->dbsize - ccb->len; if (ulogIsDebug()) nplDebug("look at datalen %d", datalen); pfrag = ds_alloc(); pfrag->seqno = pdh->seqno; pfrag->fragnum = pdh->dbno; pfrag->recsiz = datalen; pfrag->offset = heapcount; pfrag->next = NULL; /*memcpy(memheap+heapcount,prodmmap+dataoff,datalen); MD5Update(md5ctxp, (unsigned char *)(memheap+heapcount), datalen); test_deflate(compr,comprLen,(unsigned char *)(memheap+heapcount), datalen);*/ if (GOES == 1) { if (pfrag->fragnum > 0) { if ((pfrag->fragnum != prod.tail->fragnum + 1) || (pfrag->seqno != prod.seqno)) { nplError("Missing GOES fragment in sequence, " "last %d/%d this %d/%d\0", prod.tail->fragnum, prod.seqno, pfrag->fragnum, pfrag->seqno); ds_free(); prod.head = NULL; prod.tail = NULL; continue; } if ((PNGINIT != 1) && (!PROD_COMPRESSED)) { nplError("failed pnginit %d %d %s", sbn->datastream, psh->pcat, PROD_NAME); continue; } if (pdh->records_per_block < 1) { nplError("records_per_block %d blocks_per_record %d " "nx %d ny %d", pdh->records_per_block, pdh->blocks_per_record, pdb->nx, pdb->ny); nplError("source %d sector %d channel %d", pdb->source, pdb->sector, pdb->channel); nplError("nrec %d recsize %d date %02d%02d%02d %02d%02d " "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month, pdb->day, pdb->hour, pdb->minute, pdb->second, pdb->sechunds); nplError("pshname %s", psh->pname); } if (!PROD_COMPRESSED) { for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) { if (ulogIsDebug()) nplDebug("png write nscan %d", nscan); if (nscan >= pdh->records_per_block) { nplError("nscan exceeding records per block %d [%d " "%d %d]", pdh->records_per_block, nscan, pdb->nx, pdh->dbsize); } else { pngwrite(prodmmap + dataoff + (nscan * pdb->nx)); } } } else { memcpy(memheap + heapcount, prodmmap + dataoff, datalen); MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount), datalen); heapcount += datalen; } } else { if (!PROD_COMPRESSED) { png_set_memheap(memheap, md5ctxp); png_header(prodmmap + dataoff, datalen); /* * Add 1 to number of scanlines, image ends with * f0f0f0f0... */ pngout_init(pdb->nx, pdb->ny + 1); PNGINIT = 1; } else { memcpy(memheap + heapcount, prodmmap + dataoff, datalen); MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount), datalen); heapcount += datalen; } nplNotice("records_per_block %d blocks_per_record %d nx %d ny %d", pdh->records_per_block, pdh->blocks_per_record, pdb->nx, pdb->ny); nplNotice("source %d sector %d channel %d", pdb->source, pdb->sector, pdb->channel); nplNotice("nrec %d recsize %d date %02d%02d%02d %02d%02d " "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month, pdb->day, pdb->hour, pdb->minute, pdb->second, pdb->sechunds); nplNotice("pshname %s", psh->pname); } deflen = 0; } else { /* * test_deflate(memheap+heapcount,heapsize-heapcount,(unsigned char * *)(prodmmap+dataoff),datalen,&deflen); */ /* If the product already has a FOS trailer, don't add * another....this will match what pqing(SDI) sees */ if ((prod.nfrag != 0) && (prod.tail != NULL)) { if ((pfrag->fragnum != prod.tail->fragnum + 1) || (pfrag->seqno != prod.seqno)) { nplError("Missing fragment in sequence, last %d/%d this " "%d/%d\0", prod.tail->fragnum, prod.seqno, pfrag->fragnum, pfrag->seqno); ds_free(); prod.head = NULL; prod.tail = NULL; continue; } } if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) { char testme[4]; while (datalen > 4) { memcpy(testme, prodmmap + (dataoff + datalen - 4), 4); if (memcmp(testme, FOS_TRAILER, 4) == 0) { datalen -= 4; if (ulogIsDebug()) nplDebug("removing FOS trailer from %s", PROD_NAME); } else { break; } } } if (heapcount + datalen > heapsize) { /* * this above wasn't big enough heapsize = * prodalloc(psh->frags,4000+15,&memheap); */ nplError("Error in heapsize %d product size %d [%d %d], Punt!\0", heapsize, (heapcount + datalen), heapcount, datalen); continue; } memcpy(memheap + heapcount, prodmmap + dataoff, datalen); deflen = datalen; MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount), deflen); } pfrag->recsiz = deflen; /*heapcount += datalen;*/ heapcount += deflen; if (prod.head == NULL) { prod.head = pfrag; prod.tail = pfrag; } else { prod.tail->next = pfrag; prod.tail = pfrag; } if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) { if (GOES == 1) { if (PNGINIT == 1) { pngout_end(); heapcount = png_get_prodlen(); } else { if (ulogIsDebug()) nplDebug("GOES product already compressed %d", heapcount); } } if (ulogIsVerbose()) nplInfo("we should have a complete product %ld %ld/%ld %ld /heap " "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum, (long) heapcount); if ((NWSTG == 1) && (heapcount > 4)) { cnt = 4; /* number of bytes to add for TRAILER */ /* * Do a DDPLUS vs HDS check for NWSTG channel only */ if (sbn->datastream == 5) { /* nwstg channel */ switch (psh->pcat) { case 1: case 7: /* Do a quick check for non-ascii text products */ if (!prod_isascii(PROD_NAME, memheap, heapcount)) psh->pcat += 100; /* call these HDS */ /* else { ** call these DDPLUS ** if (memheap[heapcount-1] == FOS_TRAILER[3]) ** ETX check ** cnt = 0; ** no need to add extra ETX pqing doesn't see it ** } */ break; } } if (cnt > 0) { memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt); MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount), cnt); heapcount += cnt; } } process_prod(prod, PROD_NAME, memheap, heapcount, md5ctxp, ldmProdQueue, psh, sbn); ds_free(); prod.head = NULL; prod.tail = NULL; PNGINIT = 0; } else { if (ulogIsDebug()) nplDebug("processing record %ld [%ld %ld]", prod.seqno, prod.nfrag, pfrag->fragnum); if ((pdh->transtype & 4) > 0) { nplError("Hmmm....should call completed product %ld [%ld %ld]", prod.seqno, prod.nfrag, pfrag->fragnum); } } IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize); if (ulogIsDebug()) nplDebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen, deflen); } if (fd != -1) (void)close(fd); exit(0); }
/* * Sends a list of files to the LDM as data-products. * * Arguments: * ldmProxy The LDM proxy data-structure. * offer The description of the class of data-products that this * process is willing to send. * origin The identifier of the host that created the * data-products (typically the host running this program). * seq_start The starting value of the data-product sequence number. * nfiles The number of files to send. * filenames The pathnames of the files to send. * * Returns: * 0 Success. * SYSTEM_ERROR O/S failure. "log_add()" called. * CONNECTION_ABORTED The connection was aborted. "log_add()" * called. */ static int ldmsend( LdmProxy* ldmProxy, prod_class_t* offer, char* origin, int seq_start, int nfiles, char* filenames[]) { int status = 0; char* filename; int fd; struct stat statb; prod_info info; MD5_CTX* md5ctxp = NULL; prod_class_t* want; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if (md5ctxp == NULL) { log_syserr_q("new_md5_CTX failed"); return SYSTEM_ERROR; } status = lp_hiya(ldmProxy, offer, &want); if (status != 0) { status = CONNECTION_ABORTED; } else { /* These members are constant over the loop. */ info.origin = origin; info.feedtype = offer->psa.psa_val->feedtype; for (info.seqno = seq_start; exitIfDone(1) && nfiles > 0; filenames++, nfiles--, info.seqno++) { filename = *filenames; info.ident = filename; /* * ?? This could be the creation time of the file. */ (void) set_timestamp(&info.arrival); /* * Checks 'arrival', 'feedtype', and 'ident' * against what the other guy has said he wants. */ if (!prodInClass(offer, &info)) { log_info_q("Not going to send %s", filename); continue; } if (!prodInClass(want, &info)) { log_info_q("%s doesn't want %s", lp_host(ldmProxy), filename); continue; } fd = open(filename, O_RDONLY, 0); if (fd == -1) { log_syserr_q("open: %s", filename); continue; } if (fstat(fd, &statb) == -1) { log_syserr_q("fstat: %s", filename); (void) close(fd); continue; } log_info_q("Sending %s, %d bytes", filename, statb.st_size); /* These members, and seqno, vary over the loop. */ if (fd_md5(md5ctxp, fd, statb.st_size, info.signature) != 0) { (void) close(fd); continue; } if (lseek(fd, 0, SEEK_SET) == (off_t)-1) { log_syserr_q("rewind: %s", filename); (void) close(fd); continue; } info.sz = (u_int)statb.st_size; (void)exitIfDone(1); status = send_product(ldmProxy, fd, &info); (void) close(fd); if (0 != status) { log_add("Couldn't send file \"%s\" to LDM", filename); break; } } /* file loop */ if (lp_flush(ldmProxy)) log_add("Couldn't flush connection"); free_prod_class(want); } /* HIYA succeeded */ free_MD5_CTX(md5ctxp); return status; }
int main(int ac, char *av[]) { int logfd; int width; int ready; unsigned long idle; fd_set readfds; fd_set exceptfds; struct timeval timeo; const char* const progname = basename(av[0]); unsigned long maxProductSize = DEFAULT_MAX_PRODUCT_SIZE; /* * Setup default logging before anything else. */ (void)log_init(progname); feedtype = whatami(av[0]); { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 0; /* stops getopt() from printing to stderr */ usePil = 1; useNex = 1; while ((ch = getopt(ac, av, ":vxcni5Nl:b:p:P:T:q:r:f:s:")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'c': chkflag = CHK_CHECK; break; case 'n': chkflag = CHK_DONT; break; case 'i': usePil = 0; break; case 'N': useNex = 0; break; case '5': skipLeadingCtlString = 0; break; case 'l': { (void)log_set_destination(optarg); break; } case 'b': baud = optarg; break; case 'p': parity = optarg; break; #if NET case 'P': *((int *)&server_port) = atoi(optarg); /* cast away const */ if(server_port <= 0 || server_port > 65536) { log_error("Invalid server port: \"%s\"", optarg); usage(progname); } break; case 'T': reset_secs = atoi(optarg); if(reset_secs < 0) { log_add("Invalid timeout: \"%s\"", optarg); usage(progname); } break; #endif /* NET */ case 's': { unsigned long size; int nbytes; if (sscanf(optarg, "%lu %n", &size, &nbytes) != 1 || optarg[nbytes] != 0 || 1 > size) { log_error("Invalid maximum data-product size: \"%s\"", optarg); usage(progname); } maxProductSize = size; break; } case 'q': setQueuePath(optarg); break; case 'r': rawfname = optarg; break; case 'f': { feedtypet type; type = atofeedtypet(optarg); if(type != NONE) { feedtype = type; if(!parity && !baud) setFeedDefaults(type); } } break; case '?': { log_add("Unknown option: \"%c\"", optopt); usage(progname); break; } case ':': /*FALLTHROUGH*/ default: log_add("Missing argument for option: \"%c\"", optopt); usage(progname); break; } /* last arg, feedfname, is required */ if(ac - optind != 1) { log_add("Wrong number of operands: %d", ac - optind); usage(progname); } (void)strncat(feedfname, av[optind], sizeof(feedfname)-6); } pqpath = getQueuePath(); log_notice("Starting Up"); log_debug(PACKAGE_VERSION); /* * register exit handler */ if(atexit(cleanup) != 0) { log_syserr("atexit"); return 1; } /* * set up signal handlers */ set_sigactions(); /* * open the product queue, unless we were invoked as "feedtest" */ if(strcmp(progname, "feedtest") != 0) { if((ready = pq_open(pqpath, PQ_DEFAULT, &pq))) { if (PQ_CORRUPT == ready) { log_error("The product-queue \"%s\" is inconsistent\n", pqpath); } else { log_error("pq_open: \"%s\" failed: %s", pqpath, strerror(ready)); } return 1; } } /* * who am i, anyway */ (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; /* * open the feed */ if(!(*feedfname == '-' && feedfname[1] == 0) && logfd != 0) (void) close(0); if(open_feed(feedfname, &ifd, maxProductSize) != ENOERR) return 1; if(usePil == 1) { if ((feedtype & DDS)||(feedtype & PPS)||(feedtype & IDS)|| (feedtype & HRS)) { usePil = 1; log_info("Creating AFOS-like pil tags\0"); } else { usePil = 0; } } if (feedtype & HDS) { if(chkflag == CHK_CHECK || (isatty(ifd) && chkflag != CHK_DONT)) setTheScanner(scan_wmo_binary_crc); else setTheScanner(scan_wmo_binary); } else if (feedtype == ( DDPLUS | IDS ) ) { /* this is the combined NOAAPORT fos-alike. We know these have the 4 byte start and end sequences. Using the binary scanner ensures that we don't stop on an arbitray embedded CTRL-C */ log_notice("Note: Using the wmo_binary scanner for SDI ingest\0"); setTheScanner (scan_wmo_binary); } else if (feedtype & (NMC2 | NMC3)) { setTheScanner(scan_wmo_binary); } else if (feedtype == AFOS) { prod_stats = afos_stats; setTheScanner(scan_afos); } else if (feedtype == FAA604) { prod_stats = faa604_stats; if(chkflag == CHK_CHECK || (isatty(ifd) && chkflag != CHK_DONT && parity != NULL && *parity != 'n') ) { setTheScanner(scan_faa604_parity); } else { setTheScanner(scan_faa604); } } else { if(chkflag == CHK_CHECK || (isatty(ifd) && chkflag != CHK_DONT && parity != NULL && *parity != 'n') ) { setTheScanner(scan_wmo_parity); } else { setTheScanner(scan_wmo); } } /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { log_syserr("new_md5_CTX failed"); return 1; } /* * Main Loop */ idle = 0; while(exitIfDone(0)) { #if NET if (INPUT_IS_SOCKET) { if (port_error) { /* * lost connection => close */ if (ifd >= 0) { if(feed_close) (*feed_close)(ifd); ifd = -1; } port_error = 0; sleep (2); /* allow things to settle down */ continue; } } #endif if(stats_req) { log_notice("Statistics Request"); if(pq != NULL) { off_t highwater = 0; size_t maxregions = 0; (void) pq_highwater(pq, &highwater, &maxregions); log_notice(" Queue usage (bytes):%8ld", (long)highwater); log_notice(" (nregions):%8ld", (long)maxregions); } log_notice(" Idle: %8lu seconds", idle); #if NET if (INPUT_IS_SOCKET) { log_notice(" Timeout: %8d", reset_secs); } #endif log_notice("%21s: %s", "Status", (ifd < 0) ? "Not connected or input not open." : "Connected."); (*prod_stats)(); (*feed_stats)(); stats_req = 0; } #if NET if (INPUT_IS_SOCKET) { if (ifd < 0) { /* Attempt reconnect */ static int retries = 0; if (retries > MAX_RETRIES) { log_error ("maximum retry attempts %d, aborting", MAX_RETRIES); done = !0; continue; } /* Try to reopen on tcp read errors */ log_notice("Trying to re-open connection on port %d", server_port); ++retries; if(open_feed(feedfname, &ifd, maxProductSize) != ENOERR) { log_notice ("sleeping %d seconds before retry %d", retries * RETRY_DELAY, retries+1); sleep (retries * RETRY_DELAY); continue; } retries = 0; } } #endif /* NET */ timeo.tv_sec = 3; timeo.tv_usec = 0; FD_ZERO(&readfds); FD_ZERO(&exceptfds); FD_SET(ifd, &readfds); FD_SET(ifd, &exceptfds); width = ifd + 1; ready = select(width, &readfds, 0, &exceptfds, &timeo); if(ready < 0 ) { /* handle EINTR as a special case */ if(errno == EINTR) { errno = 0; continue; } log_syserr("select"); return 1; } /* else */ #if 0 if (FD_ISSET(ifd, &exceptfds)) { log_error("Exception on input fd %d, select returned %d", ifd, ready); } #endif if(ready > 0) { /* do some work */ if(FD_ISSET(ifd, &readfds) || FD_ISSET(ifd, &exceptfds)) { idle = 0; if(feedTheXbuf(ifd) != ENOERR) { #if NET if (INPUT_IS_SOCKET) { port_error = !0; continue; } /* else */ #endif /* NET */ done = !0; } FD_CLR(ifd, &readfds); FD_CLR(ifd, &exceptfds); } else { log_error("select returned %d but ifd not set", ready); idle += timeo.tv_sec; } } else /* ready == 0 */ { idle += timeo.tv_sec; #if NET if (INPUT_IS_SOCKET) { /* VOODOO * This is necessary to stimulate * 'Connection reset by peer' * when the Portmaster goes down and comes * back up. */ static char zed[1] = {0}; if(write(ifd, zed, sizeof(zed)) < 0) { port_error = !0; continue; } } #endif } #if NET if (INPUT_IS_SOCKET) { if ((reset_secs > 0) && (idle >= reset_secs)) { log_notice("Idle for %ld seconds, reconnecting", idle); /* force reconnect */ port_error = !0; idle = 0; continue; } } #endif /* NET */ (void) scanTheXbuf(); } return 0; }
int main( int ac, char *av[] ) { char *progname = av[0]; char *logfname; int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; logfname = "-"; /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv("LDMPQFNAME"); if(ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) | LOG_MASK(LOG_NOTICE)); opterr = 1; while ((ch = getopt(ac, av, "ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': logmask |= LOG_MASK(LOG_INFO); break; case 'x': logmask |= LOG_MASK(LOG_DEBUG); break; case 'l': logfname = optarg; break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case '?': usage(progname); break; } ac -= optind; av += optind ; if(ac < 1) usage(progname); (void) setulogmask(logmask); } /* * Set up error logging */ (void) openulog(ubasename(progname), LOG_NOTIME, LOG_LDM, logfname); /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strcpy(myname, ghostname()); /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { serror("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; #ifndef NO_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == NULL) { serror("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else /*!NO_MMAP*/ status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("fd_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } index = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &index); if(status != ENOERR) { serror("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { serror("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, index); index = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into "index" region */ if (status != ENOERR) { (void)pqe_discard(pq, index); index = PQE_NONE; } } /* "index" region allocated */ #endif /*!NO_MMAP*/ (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
int main (int ac, char *av[]) { char *progname = av[0]; int status; int seq_start = 0; stat_info *sinfo, *shead = NULL, *slast = NULL; int statusoff=0; /* * Set up error logging */ if (log_init(progname)) { log_syserr("Couldn't initialize logging module"); exit(1); } const char* pqfname = getQueuePath(); /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv ("LDMPQFNAME"); if (ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 1; while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': if (log_set_destination(optarg)) { log_syserr("Couldn't set logging destination to \"%s\"", optarg); exit(1); } break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi (optarg); break; case 'f': feedtype = atofeedtypet (optarg); if (feedtype == NONE) { fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg); usage (progname); } break; case 'S': statusoff=1; break; case '?': usage (progname); break; } setQueuePath(pqfname); ac -= optind; av += optind; if (ac < 1) usage (progname); } /* * register exit handler */ if (atexit (cleanup) != 0) { log_syserr ("atexit"); exit (1); } /* * set up signal handlers */ set_sigactions (); /* * who am i, anyway */ (void) strcpy (myname, ghostname ()); /* * open the product queue */ if ((status = pq_open (pqfname, PQ_DEFAULT, &pq))) { if (status > 0) { log_add_syserr("\"%s\" failed", pqfname); log_flush_error(); } else { log_error_q("\"%s\" failed: %s", pqfname, "Internal error"); } exit (2); } { char *filename; int fd; struct stat statb; product prod; unsigned char *prodmmap; MD5_CTX *md5ctxp = NULL; int gversion; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX (); if (md5ctxp == NULL) { log_syserr ("new_md5_CTX failed"); exit (6); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; prod.info.seqno = seq_start; /* * Open the file to be inserted and process */ while (ac > 0) { long insert_sum = 0; long sinfo_cnt = 0; long stat_size = 0; filename = *av; av++; ac--; log_notice_q ("open and memorymap %s\0", filename); fd = open (filename, O_RDONLY, 0); if (fd == -1) { log_syserr ("open: %s", filename); continue; } if (fstat (fd, &statb) == -1) { log_syserr ("fstat: %s", filename); (void) close (fd); continue; } if ((prodmmap = (unsigned char *) mmap (0, statb.st_size, PROT_READ, MAP_PRIVATE, fd, 0)) == MAP_FAILED) { log_syserr ("allocation failed"); } else { int GRIBDONE = 0; off_t griboff = 0; size_t griblen = 0; log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size); while (!GRIBDONE) { log_debug("griboff %d\0", (int) griboff); /* get offset of next grib product */ status = get_grib_info (prodmmap, statb.st_size, &griboff, &griblen, &gversion); switch (status) { case 0: prod.data = prodmmap + griboff; prod.info.sz = griblen; /* * revised MD5 calculation...using filename * to allow duplicate products in different files. */ MD5Init (md5ctxp); MD5Update (md5ctxp, (void *)filename, strlen(filename)); /*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/ if ( prod.info.sz > 10000 ) MD5Update (md5ctxp, (void *)prod.data, 10000); else MD5Update (md5ctxp, (void *)prod.data, prod.info.sz); MD5Final (prod.info.signature, md5ctxp); /*if (mm_md5 (md5ctxp, prod.data, prod.info.sz, prod.info.signature) != 0) { log_error_q ("could not compute MD5\0"); } else { */ prod.info.ident = (char *) malloc (KEYSIZE + 1); get_gribname (gversion, prod.data, prod.info.sz, filename, prod.info.seqno, prod.info.ident); /* * Do the deed */ status = set_timestamp (&prod.info.arrival); if (status != ENOERR) { log_add_syserr("could not set timestamp"); log_flush_error(); } /* * Insert the product */ status = pq_insert (pq, &prod); log_info_q ("%d %s\0", status, prod.info.ident); if ( status == ENOERR ) insert_sum += prod.info.sz; if (! statusoff ) { /* * Log this status */ sinfo_cnt++; sinfo = (stat_info *)malloc(sizeof(stat_info)); sinfo->insertstatus = status; sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1); strcpy(sinfo->prodname, prod.info.ident); sinfo->seqno = prod.info.seqno; sinfo->prodsz = prod.info.sz; sinfo->next = NULL; stat_size += strlen(sinfo->prodname); if(shead == NULL) { shead = sinfo; slast = sinfo; } else { slast->next = sinfo; slast = sinfo; } } /*}*/ griboff += griblen; prod.info.seqno++; break; case -1: GRIBDONE = 1; break; case -2: log_error_q ("truncated grib file at: %d", prod.info.seqno); GRIBDONE = 1; break; case -7: log_error_q ("End sequence 7777 not found where expected: %d", prod.info.seqno); griboff += griblen; log_error_q("resume looking at %d\0",griboff); break; default: log_error_q ("unknown error %d\0", status); griboff += griblen; if (griboff >= statb.st_size) GRIBDONE = 1; break; } if (griboff >= statb.st_size) GRIBDONE = 1; } log_notice_q ("munmap\0"); (void) munmap ((void *)prodmmap, statb.st_size); if ( stat_size != 0 ) /* * Add a status message to product queue */ { char *statusmess; log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt); statusmess = calloc((30 * sinfo_cnt) + stat_size + strlen(filename) + 128, sizeof(char)); if(statusmess == NULL) { log_syserr("could not malloc status message %ld\0", stat_size); } else { char tmpprod[512]; sinfo = shead; slast = NULL; status = set_timestamp(&prod.info.arrival); /* ctime ends with \n\0" */ sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n", filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec), insert_sum,(long)statb.st_size); while(sinfo != NULL) { sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus, sinfo->seqno,sinfo->prodsz,sinfo->prodname); strcat(statusmess,tmpprod); slast = sinfo; sinfo = sinfo->next; free(slast->prodname); free(slast); } shead = NULL; sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno); prod.info.ident = tmpprod; prod.data = statusmess; prod.info.sz = strlen(statusmess); status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); status = set_timestamp(&prod.info.arrival); status = pq_insert(pq, &prod); if(log_is_enabled_info) log_info_q("%s", s_prod_info(NULL, 0, &prod.info, log_is_enabled_debug)) ; free(statusmess); prod.info.seqno++; } } } (void) close (fd); } } exit(0); }
static int ldmsend(CLIENT *clnt, prod_class_t* clssp, char* origin, int seq_start, int nfiles, char* filenames[]) { int status = 0; char *filename; int fd; struct stat statb; prod_info info; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { status = errno; serror("new_md5_CTX failed"); return status; } status = (*hiya)(clnt, &clssp); if(status != 0) return status; /* These members are constant over the loop. */ info.origin = origin; info.feedtype = clssp->psa.psa_val->feedtype; for( info.seqno = seq_start; exitIfDone(1) && nfiles > 0; filenames++, nfiles--, info.seqno++) { filename = *filenames; info.ident = filename; /* * ?? This could be the creation time of the file. */ (void) set_timestamp(&info.arrival); /* * Checks 'arrival', 'feedtype', and 'ident' * against what the other guy has said he wants. */ if(!prodInClass(clssp, &info)) { uinfo("%s doesn't want %s", remote, filename); continue; } fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); continue; } uinfo("Sending %s, %d bytes", filename, statb.st_size); /* These members, and seqno, vary over the loop. */ if(fd_md5(md5ctxp, fd, statb.st_size, info.signature) != 0) { (void) close(fd); continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); continue; } (void)exitIfDone(1); info.sz = (u_int)statb.st_size; (*send_product)(clnt, fd, &info); (void) close(fd); } if (exitIfDone(1) && NULL != nullproc && NULL == (*nullproc)(NULL, clnt)) { uerror("%s: NULLPROC failure: %s", remote, clnt_errmsg(clnt)); status = 1; } free_MD5_CTX(md5ctxp); return status; }
/* * Opens the configuration file. Any previously opened configuration file will * be closed. The ulog(3) facility will be used to log any problems. * * pathname The pathname of the file. (in) * * Returns: * pq.h:ENOERR if success. * errno.h:ENOMEM if out of memory. * errno.h:EINVAL if the pathname is NULL or if the configuration file was * invalid. * (else) <errno.h> error code. */ int cfOpen(const char* pathname) { int status; if (pathname == NULL) { log_error_q("NULL pathname"); status = EINVAL; } else { if (currState != CLOSED) cfClose(); md5 = new_MD5_CTX(); if (saxer.startDocument != myStartDocument) { saxer.startDocument = myStartDocument; saxer.startElement = myStartElement; saxer.getEntity = myGetEntity; saxer.characters = myCharacters; saxer.endElement = myEndElement; saxer.endDocument = myEndDocument; saxer.warning = xmlWarning; saxer.error = xmlError; saxer.fatalError = xmlError; } (void)xmlSAXUserParseFile(&saxer, NULL, pathname); if (currState != READY) { cfClose(); status = EINVAL; } else { prodData = realloc(prodData, prodSize); if (prodData == NULL) { log_syserr_q("Couldn't allocate %u bytes for product data", prodSize); status = ENOMEM; } else { (void)memset(prodData, 'x', prodSize); prodInfo.origin = (char*)ghostname(); prodInfo.feedtype = feedType; prodInfo.ident = prodId; prodInfo.sz = prodSize; prodXdrLen = xlen_prod_i(&prodInfo); prodXdrBuf = realloc(prodXdrBuf, prodXdrLen); if (prodXdrBuf == NULL) { log_syserr_q("Couldn't allocate %lu bytes for product XDR " "buffer", prodXdrLen); status = errno; } else { prod.data = prodData; status = ENOERR; } if (status != ENOERR) { free(prodData); prodData = NULL; } } } } return status; }