void er_wmsg ( char *errgrp, int *numerr, char *errstr, int *iret) { *iret = 0; if ( *numerr != 0 ) nplError("[%s %d] %s",errgrp,*numerr,errstr); else if(ulogIsVerbose()) nplInfo("[%s %d] %s",errgrp,*numerr,errstr); }
/*ARGSUSED*/ static int addtostats(const prod_info *infop, const void *datap, void *xprod, size_t size, void *notused) { struct timeval tv; pq_ctimestamp(pq, &tv); if(tvIsNone(tv)) tv = TS_ZERO; if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, infop, ulogIsDebug())); binstats(infop, &tv); return 0; }
int prod_retrans_abort_entry (ACQ_TABLE *p_acqtable, long prod_seqno, int err_cause) { static char FNAME[]="retrans_abort_entry"; int index_value; /* index for prod_seqno */ int retrans_table_typ; /* retrans table type */ PROD_RETRANS_ENTRY *p_retrans_entry; PROD_RETRANS_ENTRY_INFO *p_retrans_entry_info; if(prod_retrans_get_addr(p_acqtable->proc_base_channel_type_last, p_prod_retrans_table, &p_retrans_entry_info, &p_retrans_entry, &retrans_table_typ) < 0){ uerror("%s ignore abort \n",FNAME); return(ERROR); } /* Now get the index value */ index_value = prod_seqno % p_retrans_entry_info->numb_entries; if(index_value < 0) { index_value = -index_value; } if(index_value >= p_retrans_entry_info->numb_entries) { index_value = 0; } if(ulogIsVerbose ()){ uinfo("%s ok abort %s tbl[%d]=%ld\n",FNAME, GET_SBN_TYP_NAME(p_acqtable->proc_base_channel_type_last), index_value, p_retrans_entry[index_value].prod_seqno); } prod_retrans_update_entry(p_acqtable, (BUFF_HDR *)NULL, p_retrans_entry_info, &p_retrans_entry[index_value], index_value, prod_seqno, p_acqtable->proc_prod_run_id, RETRANS_ENTRY_FLAG_AVAIL, err_cause); return(0); }
static int split_prod(const prod_info *infop, const void *datap, void *xprod, size_t size, void *vp) { size_t *nsp = (size_t *)vp; int ns; if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, infop, ulogIsDebug())); ns = surf_split(infop, datap, doOne); nprods++; (void)kill(SIGCONT, act_pid); if(nsp != NULL && ns >= 0) *nsp = (size_t)ns; return 0; }
static int doOne(const prod_info *infop, const void *datap) { struct product prod; int status = ENOERR; if(ulogIsDebug()) udebug("%s", s_prod_info(NULL, 0, infop, 1)); prod.info = *infop; prod.data = (void *)datap; /* cast away const */ nsplit++; /* ?? Do it here on only on success ?? */ status = pq_insertNoSig(opq, &prod); if(status == ENOERR) { return status; /* Normal return */ } /* else */ if(status == PQUEUE_DUP) { ndups++; if(ulogIsVerbose()) uinfo("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())); return status; } /* else, error */ uerror("pq_insert: %s\n", strerror(status)); return status; }
hiya_reply_t* hiya_6_svc( prod_class_t *offered, struct svc_req *rqstp) { const char* const pqfname = getQueuePath(); static hiya_reply_t reply; SVCXPRT * const xprt = rqstp->rq_xprt; struct sockaddr_in *upAddr = (struct sockaddr_in*) svc_getcaller(xprt); const char *upName = hostbyaddr(upAddr); int error; int isPrimary; unsigned int maxHereis; static prod_class_t *accept; /* * Open the product-queue for writing. It will be closed by cleanup() * during process termination. */ if (pq) { (void) pq_close(pq); pq = NULL; } error = pq_open(pqfname, PQ_DEFAULT, &pq); if (error) { err_log_and_free(ERR_NEW2(error, NULL, "Couldn't open product-queue \"%s\" for writing: %s", pqfname, PQ_CORRUPT == error ? "The product-queue is inconsistent" : strerror(error)), ERR_FAILURE); svcerr_systemerr(xprt); svc_destroy(xprt); exit(error); } /* else */ error = down6_init(upName, upAddr, pqfname, pq); if (error) { uerror("Couldn't initialize downstream LDM"); svcerr_systemerr(xprt); svc_destroy(xprt); exit(error); } else { uinfo("Downstream LDM initialized"); } /* * The previous "accept" is freed here -- rather than freeing the * soon-to-be-allocated "accept" at the end of its block -- because it can * be used in the reply. */ if (accept) { free_prod_class(accept); /* NULL safe */ accept = NULL; } error = lcf_reduceToAcceptable(upName, inet_ntoa(upAddr->sin_addr), offered, &accept, &isPrimary); maxHereis = isPrimary ? UINT_MAX : 0; if (error) { serror("Couldn't validate HIYA"); svcerr_systemerr(xprt); svc_destroy(xprt); exit(error); } else { if (ulogIsDebug()) udebug("intersection: %s", s_prod_class(NULL, 0, accept)); if (accept->psa.psa_len == 0) { uwarn("Empty intersection of HIYA offer from %s (%s) and ACCEPT " "entries", upName, s_prod_class(NULL, 0, offered)); svcerr_weakauth(xprt); svc_destroy(xprt); exit(0); } else { error = down6_set_prod_class(accept); if (error) { if (DOWN6_SYSTEM_ERROR == error) { serror("Couldn't set product class: %s", s_prod_class(NULL, 0, accept)); } else { uerror("Couldn't set product class: %s", s_prod_class(NULL, 0, accept)); } svcerr_systemerr(xprt); svc_destroy(xprt); exit(EXIT_FAILURE); } /* else */ if (clss_eq(offered, accept)) { unotice("hiya6: %s", s_prod_class(NULL, 0, offered)); reply.code = OK; reply.hiya_reply_t_u.max_hereis = maxHereis; } else { if (ulogIsVerbose()) { char off[512]; char acc[512]; (void) s_prod_class(off, sizeof(off), offered), (void) s_prod_class( acc, sizeof(acc), accept); uinfo("hiya6: RECLASS: %s -> %s", off, acc); } reply.code = RECLASS; reply.hiya_reply_t_u.feedPar.prod_class = accept; reply.hiya_reply_t_u.feedPar.max_hereis = maxHereis; } } /* product-intersection != empty set */ } /* successful acl_check_hiya() */ return &reply; }
int main( int ac, char *av[] ) { const char* const pqfname = getQueuePath(); const char* const progname = ubasename(av[0]); int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, /* all files inserted successfully */ exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_dup = 4, /* input-file already in product-queue */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; #if !USE_MMAP pqeIndex = PQE_NONE; #endif { extern int optind; extern int opterr; extern char *optarg; int ch; (void) openulog(progname, LOG_NOTIME, LOG_LDM, "-"); (void) setulogmask(LOG_UPTO(LOG_NOTICE)); opterr = 0; /* Suppress getopt(3) error messages */ while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': (void) setulogmask(getulogmask() | LOG_MASK(LOG_INFO)); break; case 'x': (void) setulogmask(getulogmask() | LOG_MASK(LOG_DEBUG)); break; case 'l': openulog(progname, ulog_get_options(), LOG_LDM, optarg); break; case 'q': setQueuePath(optarg); break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case ':': { LOG_ADD1("Option \"-%c\" requires an operand", optopt); usage(progname); } /* no break */ default: LOG_ADD1("Unknown option: \"%c\"", optopt); usage(progname); /* no break */ } ac -= optind; av += optind ; if(ac < 1) usage(progname); } /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { serror("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } #if USE_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == NULL) { serror("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case PQUEUE_BIG: uerror("Product too big for queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_infile; break; case ENOMEM: uerror("queue full?"); exitCode = exit_system; break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else // USE_MMAP above; !USE_MMAP below status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("xx_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } pqeIndex = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex); if(status != ENOERR) { serror("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { serror("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, pqeIndex); pqeIndex = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into `pqeIndex` region */ if (status != ENOERR) { (void)pqe_discard(pq, pqeIndex); pqeIndex = PQE_NONE; } } /* `pqeIndex` region allocated */ #endif (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
int main(int ac, char *av[]) { const char* pqfname = getQueuePath(); const char *opqfname = getSurfQueuePath(); const char *progname = ubasename(av[0]); char *logfname; prod_class_t clss; prod_spec spec; int status = 0; unsigned interval = DEFAULT_INTERVAL; int logoptions = (LOG_CONS|LOG_PID); double age = DEFAULT_AGE; /* these are containers for the pqact args */ char *argv[16]; int argc = 0; int toffset = TOFFSET_NONE; logfname = ""; if(set_timestamp(&clss.from) != ENOERR) /* corrected by toffset below */ { int errnum = errno; fprintf(stderr, "Couldn't set timestamp: %s", strerror(errnum)); exit(1); } clss.to = TS_ENDT; clss.psa.psa_len = 1; clss.psa.psa_val = &spec; spec.feedtype = DEFAULT_FEEDTYPE; spec.pattern = DEFAULT_PATTERN; memset(argv, 0, sizeof(argv)); argv[0] = "pqact"; argc++; { extern int optind; extern int opterr; extern char *optarg; int ch; int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) | LOG_MASK(LOG_NOTICE)); int fterr; const char *conffilename = getPqsurfConfigPath(); const char *datadir = getPqsurfDataDirPath(); usePil = 1; opterr = 1; while ((ch = getopt(ac, av, "vxl:d:f:p:q:Q:o:i:a:t:")) != EOF) switch (ch) { case 'v': argv[argc++] = "-v"; logmask |= LOG_MASK(LOG_INFO); break; case 'x': argv[argc++] = "-x"; logmask |= LOG_MASK(LOG_DEBUG); break; case 'l': argv[argc++] = "-l"; argv[argc++] = optarg; logfname = optarg; break; case 'd': datadir = optarg; break; case 'f': fterr = strfeedtypet(optarg, &spec.feedtype); if(fterr != FEEDTYPE_OK) { fprintf(stderr, "%s: %s: \"%s\"\n", av[0], strfeederr(fterr), optarg); usage(progname); } argv[argc++] = "-f"; argv[argc++] = optarg; break; case 'p': spec.pattern = optarg; /* compiled below */ break; case 'q': pqfname = optarg; break; case 'Q': opqfname = optarg; break; case 'o': toffset = atoi(optarg); if(toffset == 0 && *optarg != '0') { fprintf(stderr, "%s: invalid offset %s\n", av[0], optarg); usage(av[0]); } argv[argc++] = "-o"; argv[argc++] = optarg; break; case 'i': interval = atoi(optarg); if(interval == 0 && *optarg != '0') { fprintf(stderr, "%s: invalid interval \"%s\"\n", av[0], optarg); usage(av[0]); } /* N.B. -i just used for input queue. */ break; case 'a': age = atof(optarg); if(age < 0.) { (void) fprintf(stderr, "age (%s) must be non negative\n", optarg); usage(av[0]); } break; case 't': /* pipe_timeo */ argv[argc++] = "-t"; argv[argc++] = optarg; break; case '?': usage(progname); break; } (void) setulogmask(logmask); if (re_isPathological(spec.pattern)) { fprintf(stderr, "Adjusting pathological regular-expression: " "\"%s\"\n", spec.pattern); re_vetSpec(spec.pattern); } status = regcomp(&spec.rgx, spec.pattern, REG_EXTENDED|REG_NOSUB); if(status != 0) { fprintf(stderr, "Bad regular expression \"%s\"\n", spec.pattern); usage(av[0]); } if(ac - optind == 1) conffilename = av[optind]; argv[argc++] = "-d"; argv[argc++] = (char*)datadir; argv[argc++] = "-q"; argv[argc++] = (char*)opqfname; argv[argc++] = (char*)conffilename; age *= 3600.; } if(toffset != TOFFSET_NONE) { clss.from.tv_sec -= toffset; } else { clss.from.tv_sec -= (age - interval); } /* * Set up error logging. * N.B. log ident is the remote */ (void) openulog(progname, logoptions, LOG_LDM, logfname); unotice("Starting Up (%d)", getpgrp()); /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(1); } /* * set up signal handlers */ set_sigactions(); /* * Open the output product queue */ status = pq_open(opqfname, PQ_DEFAULT, &opq); if(status) { if (PQ_CORRUPT == status) { uerror("The output product-queue \"%s\" is inconsistent\n", opqfname); } else { uerror("pq_open failed: %s: %s\n", opqfname, strerror(status)); } exit(1); } act_pid = run_child(argc, argv); if(act_pid == (pid_t)-1) exit(1); /* * Open the input product queue */ status = pq_open(pqfname, PQ_READONLY, &pq); if(status) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open failed: %s: %s\n", pqfname, strerror(status)); } exit(1); } if(toffset == TOFFSET_NONE) { /* Jump to the end of the queue */ timestampt sav; sav = clss.from; clss.from = TS_ZERO; (void) pq_last(pq, &clss, NULL); clss.from = sav; } else { pq_cset(pq, &clss.from); } if(ulogIsVerbose()) { char buf[1984]; uinfo("%s", s_prod_class(buf, sizeof(buf), &clss)); } while(exitIfDone(0)) { if(stats_req) { dump_stats(); stats_req = 0; } status = pq_sequence(pq, TV_GT, &clss, split_prod, NULL); switch(status) { case 0: /* no error */ continue; /* N.B., other cases sleep */ case PQUEUE_END: udebug("surf: End of Queue"); break; case EAGAIN: case EACCES: udebug("Hit a lock"); break; default: uerror("pq_sequence failed: %s (errno = %d)", strerror(status), status); exit(1); break; } if(interval == 0) { break; } (void) expire(opq, interval, age); pq_suspend(interval); (void) reap_act(WNOHANG); } /* * TODO: how can we determine that pqact has finished * the work in opq? */ sleep(5); exit(0); }
/** * Executes a product-maker. * * This function is thread-compatible but not thread-safe. * * @retval (void*)0 The FIFO was closed. * @retval (void*)1 Usage failure. \c log_start() called. * @retval (void*)2 O/S failure. \c log_start() called. * @retval (void*)-1 Retransmission failure. \c log_start() called. */ void* pmStart( void* const arg) /**< [in/out] Pointer to the * product-maker to be executed */ { ProductMaker* const productMaker = (ProductMaker*)arg; int status; Fifo* const fifo = productMaker->fifo; unsigned char* buf = productMaker->buf; sbn_struct* sbn = &productMaker->sbn; pdh_struct* pdh = &productMaker->pdh; psh_struct* psh = &productMaker->psh; pdb_struct* pdb = &productMaker->pdb; ccb_struct* ccb = &productMaker->ccb; unsigned long last_sbn_seqno; unsigned long last_sbn_runno = ULONG_MAX; int PNGINIT = 0; char* memheap = NULL; MD5_CTX* md5ctxp = productMaker->md5ctxp; int logResync = 1; prodstore prod; #ifdef RETRANS_SUPPORT long cpio_addr_tmp; int cpio_fd_tmp; /* char transfer_type[10]={0};*/ int retrans_val,idx; long retrans_tbl_size; time_t orig_arrive_time; int new_key; /* shm key */ ACQ_TABLE *acq_tbl = NULL; long num_prod_discards=0; int save_prod = 1; int discard_prod = 0; long proc_orig_prod_seqno_last_save=0; #endif prod.head = NULL; prod.tail = NULL; /*** For Retranmission Purpose ***/ #ifdef RETRANS_SUPPORT if (ulogIsDebug()) udebug(" retrans_xmit_enable = %d transfer_type = %s sbn_channel_name=%s \n",retrans_xmit_enable,transfer_type,sbn_channel_name); if((retrans_xmit_enable == OPTION_ENABLE) && (!strcmp(transfer_type,"MHS") || !strcmp(transfer_type,"mhs"))){ idx = get_cpio_addr(mcastAddr); if( idx >= 0 && idx < NUM_CPIO_ENTRIES){ global_cpio_fd = cpio_tbl[idx].cpio_fd; global_cpio_addr = cpio_tbl[idx].cpio_addr; if (ulogIsDebug()) udebug("Global cpio_addr = 0x%x Global cpio_fd = %d \n",global_cpio_addr,global_cpio_fd); }else{ uerror("Invalid multicast address provided"); return (void*)-1; } retrans_tbl_size = sizeof(PROD_RETRANS_TABLE); /** Modified to setup retrans table on per channel basis - Sathya - 14-Mar'2013 **/ retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * GET_RETRANS_CHANNEL_ENTRIES(sbn_type)); /**** retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC); retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC1); retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC2); retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC3); retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_GOES_EAST); retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NOAAPORT_OPT); ****/ p_prod_retrans_table = (PROD_RETRANS_TABLE *) malloc (retrans_tbl_size); if(p_prod_retrans_table == NULL){ uerror("Unable to allocate memory for retrans table..Quitting.\n"); return (void*)-1; } if( init_retrans(&p_prod_retrans_table) < 0 ){ uerror("Error in initializing retrans table \n"); if(p_prod_retrans_table) free(p_prod_retrans_table); return (void*)-1; } GET_SHMPTR(global_acq_tbl,ACQ_TABLE,ACQ_TABLE_SHMKEY,DEBUGGETSHM); if (ulogIsDebug()) udebug("Global acquisition table = 0x%x cpio_fd = %d \n",global_acq_tbl,global_cpio_fd); acq_tbl = &global_acq_tbl[global_cpio_fd]; if (ulogIsDebug()) udebug("Obtained acquisition table = 0x%x \n",acq_tbl); /* ACQ_TABLE already initialized in acq_ldm_getshm */ /* if(init_acq_table(acq_tbl) < 0){ uerror("Unable to initialize acq table\n"); exit(-1); } */ buff_hdr = (BUFF_HDR *) malloc(sizeof(BUFF_HDR)); if(init_buff_hdr(buff_hdr) < 0){ uerror("Unalbe to initialize buffer header \n"); if(acq_tbl) free(acq_tbl); if(p_prod_retrans_table) free(p_prod_retrans_table); return (void*)-1; } acq_tbl->pid = getpid(); acq_tbl->link_id = global_cpio_fd; acq_tbl->link_addr = global_cpio_addr; if(ulogIsVerbose()){ uinfo("Initialized acq_tbl = 0x%x & buff_hdr = 0x%x pid = %d \n",acq_tbl,buff_hdr,acq_tbl->pid); uinfo("Global link id = %d Global link addr = %ld \n",acq_tbl->link_id,acq_tbl->link_addr); uinfo("acq_tbl->read_distrib_enable = 0x%x \n",acq_tbl->read_distrib_enable); } } /*** For Retranmission Purpose ***/ #endif for (;;) { unsigned char b1; long IOFF; int NWSTG; int GOES; int PROD_COMPRESSED; size_t heapcount; size_t heapsize; char PROD_NAME[1024]; int dataoff; int datalen; datastore* pfrag; int nscan; int deflen; static const char* FOS_TRAILER = "\015\015\012\003"; int cnt; /* Look for first byte == 255 and a valid SBN checksum */ if ((status = fifoRead(fifo, buf, 1)) != 0) { if (3 == status) status = 0; break; } if ((b1 = (unsigned char)buf[0]) != 255) { if (logResync) { uinfo("Trying to resync %u", b1); logResync = 0; } continue; } logResync = 1; if (fifoRead(fifo, buf + 1, 15) != 0) { if (ulogIsDebug()) udebug("couldn't read 16 bytes for sbn"); continue; } while ((status = readsbn(buf, sbn)) != 0) { if (ulogIsDebug()) udebug("Not SBN start"); IOFF = 1; while ((IOFF < 16) && ((b1 = (unsigned char) buf[IOFF]) != 255)) IOFF++; if (IOFF > 15) { break; } else { int ch; for (ch = IOFF; ch < 16; ch++) buf[ch - IOFF] = buf[ch]; if (fifoRead(fifo, buf + 16 - IOFF, IOFF) != 0) { if (ulogIsDebug()) udebug("Couldn't read bytes for SBN, resync"); break; } } } if (status != 0) { if (ulogIsDebug()) udebug("SBN status continue"); continue; } IOFF = 0; if (fifoRead(fifo, buf + 16, 16) != 0) { if (ulogIsDebug()) udebug("error reading Product Definition Header"); continue; } #ifdef RETRANS_SUPPORT /* Update acq table stats - Begin */ if(retrans_xmit_enable == OPTION_ENABLE){ buff_hdr->read_channel_type = sbn->datastream; } /* Update acq table stats - End */ #endif if (ulogIsDebug()) udebug("***********************************************"); if (last_sbn_runno != sbn->runno) { last_sbn_runno = sbn->runno; } else { unsigned long delta = sbn->seqno - last_sbn_seqno; # define MAX_SEQNO 0xFFFFFFFFu if (0 == delta || MAX_SEQNO/2 < delta) { uwarn("Retrograde packet number: previous=%lu, latest=%lu, " "difference=%lu", last_sbn_seqno, sbn->seqno, 0 == delta ? 0ul : MAX_SEQNO - delta + 1); } else { if (1 != delta) { unsigned long gap = delta - 1; uwarn("Gap in packet sequence: %lu to %lu [skipped %lu]", last_sbn_seqno, sbn->seqno, gap); (void)pthread_mutex_lock(&productMaker->mutex); productMaker->nmissed += gap; (void)pthread_mutex_unlock(&productMaker->mutex); } (void)pthread_mutex_lock(&productMaker->mutex); productMaker->npackets++; (void)pthread_mutex_unlock(&productMaker->mutex); } /* non-retrograde packet number */ } /* "last_sbn_seqno" initialized */ last_sbn_seqno = sbn->seqno; if (ulogIsVerbose()) uinfo("SBN seqnumber %ld", sbn->seqno); if (ulogIsVerbose()) uinfo("SBN datastream %d command %d", sbn->datastream, sbn->command); if (ulogIsDebug()) udebug("SBN version %d length offset %d", sbn->version, sbn->len); if (((sbn->command != 3) && (sbn->command != 5)) || (sbn->version != 1)) { uerror("Unknown sbn command/version %d PUNT", sbn->command); continue; } switch (sbn->datastream) { case 5: /* nwstg */ case 6: /* nwtg2 */ case 7: /* polarsat */ case 8: /* NOAA Weather Wire Service (NWWS) */ case 9: case 10: case 11: case 12: /* GOES-R */ case 13: /* GOES-R */ NWSTG = 1; GOES = 0; break; case 1: /* GINI GOES */ case 2: /* GINI GOES */ case 4: /* OCONUS */ NWSTG = 0; GOES = 1; break; default: uerror("Unknown NOAAport channel %d PUNT", sbn->datastream); continue; } /* End of SBN version low 4 bits */ if (readpdh(buf + IOFF + sbn->len, pdh) == -1) { uerror("problem with pdh, PUNT"); continue; } if (pdh->len > 16) { if (fifoRead(fifo, buf + sbn->len + 16, pdh->len - 16) != 0) continue; } if (ulogIsDebug()) udebug("Product definition header version %d pdhlen %d", pdh->version, pdh->len); if (pdh->version != 1) { uerror("Error: PDH transfer type %u, PUNT", pdh->transtype); continue; } else if (ulogIsDebug()) { udebug("PDH transfer type %u", pdh->transtype); } if ((pdh->transtype & 8) > 0) uerror("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 32) > 0) uerror("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 16) > 0) { PROD_COMPRESSED = 1; if (ulogIsDebug()) udebug("Product transfer flag compressed %u", pdh->transtype); } else { PROD_COMPRESSED = 0; } if (ulogIsDebug()) udebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen, pdh->pshlen); if (ulogIsDebug()) udebug("blocks per record %ld records per block %ld\n", pdh->blocks_per_record, pdh->records_per_block); if (ulogIsDebug()) udebug("product seqnumber %ld block number %ld data block size " "%ld", pdh->seqno, pdh->dbno, pdh->dbsize); /* Stop here if no psh */ if ((pdh->pshlen == 0) && (pdh->transtype == 0)) { IOFF = IOFF + sbn->len + pdh->len; continue; } #ifdef RETRANS_SUPPORT /** Update acquisition table statistics **/ if(retrans_xmit_enable == OPTION_ENABLE){ acq_tbl->read_tot_buff_read++; } #endif if (pdh->pshlen != 0) { if (fifoRead(fifo, buf + sbn->len + pdh->len, pdh->pshlen) != 0) { uerror("problem reading psh"); continue; } else { if (ulogIsDebug()) udebug("read psh %d", pdh->pshlen); } /* Timing block */ if (sbn->command == 5) { if (ulogIsDebug()) udebug("Timing block recieved %ld %ld\0", psh->olen, pdh->len); /* * Don't step on our psh of a product struct of prod in * progress. */ continue; } if (readpsh(buf + IOFF + sbn->len + pdh->len, psh) == -1) { uerror("problem with readpsh"); continue; } if (psh->olen != pdh->pshlen) { uerror("ERROR in calculation of psh len %ld %ld", psh->olen, pdh->len); continue; } if (ulogIsDebug()) udebug("len %ld", psh->olen); if (ulogIsDebug()) udebug("product header flag %d, version %d", psh->hflag, psh->version); if (ulogIsDebug()) udebug("prodspecific data length %ld", psh->psdl); if (ulogIsDebug()) udebug("bytes per record %ld", psh->bytes_per_record); if (ulogIsDebug()) udebug("Fragments = %ld category %d ptype %d code %d", psh->frags, psh->pcat, psh->ptype, psh->pcode); if (psh->frags < 0) uerror("check psh->frags %d", psh->frags); if (psh->origrunid != 0) uerror("original runid %d", psh->origrunid); if (ulogIsDebug()) udebug("next header offset %ld", psh->nhoff); if (ulogIsDebug()) udebug("original seq number %ld", psh->seqno); if (ulogIsDebug()) udebug("receive time %ld", psh->rectime); if (ulogIsDebug()) udebug("transmit time %ld", psh->transtime); if (ulogIsDebug()) udebug("run ID %ld", psh->runid); if (ulogIsDebug()) udebug("original run id %ld", psh->origrunid); #ifdef RETRANS_SUPPORT /* Update acq table stats - Begin */ if(retrans_xmit_enable == OPTION_ENABLE){ buff_hdr->buff_data_length = pdh->dbsize; if(pdh->dbno == 0) { /* Assume first block */ acq_tbl->proc_base_prod_type_last = psh->ptype; acq_tbl->proc_base_prod_cat_last = psh->pcat; acq_tbl->proc_base_prod_code_last = psh->pcode; acq_tbl->proc_prod_NCF_rcv_time = (time_t)psh->rectime; acq_tbl->proc_prod_NCF_xmit_time = (time_t)psh->transtime; if(psh->hflag & XFR_PROD_RETRANSMIT){ acq_tbl->proc_orig_prod_seqno_last = psh->seqno; acq_tbl->proc_orig_prod_run_id = psh->origrunid; if(ulogIsDebug()) udebug("ORIG SEQ# = %ld CURR SEQ#: %ld \n",acq_tbl->proc_orig_prod_seqno_last,pdh->seqno); }else{ acq_tbl->proc_orig_prod_seqno_last = 0; acq_tbl->proc_orig_prod_run_id = 0; } acq_tbl->proc_prod_run_id = psh->runid; buff_hdr->buff_datahdr_length = psh->psdl; time(&acq_tbl->proc_prod_start_time); acq_tbl->proc_tot_prods_handled++; }else{ buff_hdr->buff_datahdr_length = 0; } buff_hdr->proc_prod_seqno= pdh->seqno; buff_hdr->proc_blkno = pdh->dbno; buff_hdr->proc_sub_code = 0; buff_hdr->proc_prod_flag = pdh->transtype; acq_tbl->proc_base_channel_type_last = buff_hdr->read_channel_type; buff_hdr->proc_prod_type = acq_tbl->proc_base_prod_type_last; buff_hdr->proc_prod_code = acq_tbl->proc_base_prod_code_last; buff_hdr->proc_prod_cat = acq_tbl->proc_base_prod_cat_last; acq_tbl->proc_prod_bytes_read = buff_hdr->buff_data_length; /* Check prod_seqno for lost products */ if((buff_hdr->proc_prod_seqno - acq_tbl->proc_base_prod_seqno_last) != 1){ do_prod_lost(buff_hdr,acq_tbl); } retrans_val = prod_retrans_ck(acq_tbl, buff_hdr, &orig_arrive_time); log_buff[0] = '\0'; if((retrans_val == PROD_DUPLICATE_DISCARD) || ((retrans_val == PROD_DUPLICATE_MATCH) && (acq_tbl->proc_retransmit_ctl_flag & ENABLE_RETRANS_DUP_MATCH_DISCARD)) || ((retrans_val == PROD_DUPLICATE_NOMATCH) && (acq_tbl->proc_retransmit_ctl_flag & ENABLE_RETRANS_DUP_NOMATCH_DISCARD))){ /* Log product details and discard the product */ strcpy(log_buff,"DISCARD"); if(acq_tbl->proc_orig_prod_seqno_last != 0){ strcat(log_buff, "/RETRANS"); } log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last, buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno, buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,orig_arrive_time); save_prod = 0; acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; /* Current prod discarded and continue with next */ /*continue; */ }else{ if(retrans_val == PROD_DUPLICATE_NOMATCH){ strcpy(log_buff,"SAVE RETRANS"); log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last, buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno, buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,acq_tbl->proc_prod_start_time); } } } #endif if (prod.head != NULL) { uerror("OOPS, start of new product [%ld ] with unfinished " "product %ld", pdh->seqno, prod.seqno); #ifdef RETRANS_SUPPORT /* Request retrans when prod is partially received but before completion */ /* if there is frame error and continue with different prod then, we need */ /* to abort the old prod and clear retrans table. */ if((retrans_xmit_enable == OPTION_ENABLE) /*&& (pdh->dbno != 0)*/){ acq_tbl->proc_acqtab_prodseq_errs++; if(proc_orig_prod_seqno_last_save != acq_tbl->proc_orig_prod_seqno_last){ /* Clear retrans table for the orig prod if the previous prod is retrans */ /* of original prod */ prod_retrans_abort_entry(acq_tbl, proc_orig_prod_seqno_last_save, RETRANS_RQST_CAUSE_RCV_ERR); } prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR); /* Update Statistics */ acq_tbl->proc_tot_prods_lost_errs++; /* For now, generate retrans request only for non-imagery products */ if(!((buff_hdr->proc_prod_cat == PROD_CAT_IMAGE) && (PROD_TYPE_NESDIS_HDR_TRUE(buff_hdr->proc_prod_type)))){ generate_retrans_rqst(acq_tbl,prod.seqno , prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR); } acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; } #endif ds_free(); prod.head = NULL; prod.tail = NULL; if (PNGINIT != 0) { pngout_end(); PNGINIT = 0; } uerror("Product definition header version %d pdhlen %d", pdh->version, pdh->len); uerror("PDH transfer type %u", pdh->transtype); if ((pdh->transtype & 8) > 0) uerror("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 32) > 0) uerror("Product transfer flag error %u", pdh->transtype); uerror("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen, pdh->pshlen); uerror("blocks per record %ld records per block %ld", pdh->blocks_per_record, pdh->records_per_block); uerror("product seqnumber %ld block number %ld data block " "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize); uerror("product header flag %d", psh->hflag); uerror("prodspecific data length %ld", psh->psdl); uerror("bytes per record %ld", psh->bytes_per_record); uerror("Fragments = %ld category %d", psh->frags, psh->pcat); if (psh->frags < 0) uerror("check psh->frags %d", psh->frags); if (psh->origrunid != 0) uerror("original runid %d", psh->origrunid); uerror("next header offset %ld", psh->nhoff); uerror("original seq number %ld", psh->seqno); uerror("receive time %ld", psh->rectime); uerror("transmit time %ld", psh->transtime); uerror("run ID %ld", psh->runid); uerror("original run id %ld", psh->origrunid); } prod.seqno = pdh->seqno; prod.nfrag = psh->frags; ds_init(prod.nfrag); /* NWSTG CCB = dataoff, WMO = dataoff + 24 */ if (fifoRead(fifo, buf + sbn->len + pdh->len + pdh->pshlen, pdh->dbsize) != 0) { uerror("problem reading datablock"); continue; } if (sbn->datastream == 4) { if (psh->pcat != 3) { GOES = 0; NWSTG = 1; } } heapcount = 0; MD5Init(md5ctxp); if (GOES == 1) { if (readpdb(buf + IOFF + sbn->len + pdh->len + pdh->pshlen, psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) { uerror("Error reading pdb, punt"); continue; } (void)memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME)); if (ulogIsDebug()) udebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len, pdh->pshlen, sbn->len + pdh->len + pdh->pshlen, pdb->len); /* Data starts at first block after pdb */ ccb->len = 0; heapsize = prodalloc(psh->frags, 5152, &memheap); } if (NWSTG == 1) { memset(psh->pname, 0, sizeof(psh->pname)); if (readccb(buf + IOFF + sbn->len + pdh->len + pdh->pshlen, ccb, psh, pdh->dbsize) == -1) uerror("Error reading ccb, using default name"); if (ulogIsDebug()) udebug("look at ccb start %d %d", ccb->b1, ccb->len); if (ulogIsVerbose()) uinfo("%s", psh->pname); memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME)); heapsize = prodalloc(psh->frags, 4000 + 15, &memheap); /* * We will only compute md5 checksum on the data, 11 FOS * characters at start */ sprintf(memheap, "\001\015\015\012%03d\040\015\015\012", ((int) pdh->seqno) % 1000); heapcount += 11; if (psh->metaoff > 0) psh->metaoff = psh->metaoff + 11; } } else { /* If a continuation record...don't let psh->pcat get missed */ if ((sbn->datastream == 4) && (psh->pcat != 3)) { GOES = 0; NWSTG = 1; } ccb->len = 0; if (ulogIsDebug()) udebug("continuation record"); #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ buff_hdr->buff_data_length = pdh->dbsize; buff_hdr->buff_datahdr_length = 0; buff_hdr->proc_prod_seqno= pdh->seqno; buff_hdr->proc_blkno = pdh->dbno; buff_hdr->proc_sub_code = 0; buff_hdr->proc_prod_flag = pdh->transtype; acq_tbl->proc_base_channel_type_last = buff_hdr->read_channel_type; buff_hdr->proc_prod_type = acq_tbl->proc_base_prod_type_last; buff_hdr->proc_prod_code = acq_tbl->proc_base_prod_code_last; buff_hdr->proc_prod_cat = acq_tbl->proc_base_prod_cat_last; acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length; } #endif if ((pdh->transtype & 4) > 0) { psh->frags = 0; } if (fifoRead(fifo, buf + sbn->len + pdh->len + pdh->pshlen, pdh->dbsize) != 0) { uerror("problem reading datablock (cont)"); continue; } if (prod.head == NULL) { if (ulogIsVerbose()) uinfo("found data block before header, " "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno); continue; } } /* Get the data */ dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len; datalen = pdh->dbsize - ccb->len; if (ulogIsDebug()) udebug("look at datalen %d", datalen); pfrag = ds_alloc(); pfrag->seqno = pdh->seqno; pfrag->fragnum = pdh->dbno; pfrag->recsiz = datalen; pfrag->offset = heapcount; pfrag->next = NULL; if (GOES == 1) { if (pfrag->fragnum > 0) { if ((pfrag->fragnum != prod.tail->fragnum + 1) || (pfrag->seqno != prod.seqno)) { uerror("Missing GOES fragment in sequence, " "last %d/%d this %d/%d\0", prod.tail->fragnum, prod.seqno, pfrag->fragnum, pfrag->seqno); #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ acq_tbl->proc_acqtab_prodseq_errs++; do_prod_mismatch(acq_tbl,buff_hdr); acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; } #endif ds_free(); prod.head = NULL; prod.tail = NULL; continue; } if ((PNGINIT != 1) && (!PROD_COMPRESSED)) { uerror("failed pnginit %d %d %s", sbn->datastream, psh->pcat, PROD_NAME); continue; } if (pdh->records_per_block < 1) { uerror("records_per_block %d blocks_per_record %d " "nx %d ny %d", pdh->records_per_block, pdh->blocks_per_record, pdb->nx, pdb->ny); uerror("source %d sector %d channel %d", pdb->source, pdb->sector, pdb->channel); uerror("nrec %d recsize %d date %02d%02d%02d %02d%02d " "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month, pdb->day, pdb->hour, pdb->minute, pdb->second, pdb->sechunds); uerror("pshname %s", psh->pname); } if (!PROD_COMPRESSED) { for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) { if (ulogIsDebug()) udebug("png write nscan %d", nscan); if (nscan >= pdh->records_per_block) { uerror("nscan exceeding records per block %d [%d " "%d %d]", pdh->records_per_block, nscan, pdb->nx, pdh->dbsize); } else { pngwrite(buf + dataoff + (nscan * pdb->nx)); } } } else { memcpy(memheap + heapcount, buf + dataoff, datalen); MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount), datalen); heapcount += datalen; } } else { if (!PROD_COMPRESSED) { png_set_memheap(memheap, md5ctxp); png_header(buf + dataoff, datalen); /* * Add 1 to number of scanlines, image ends with * f0f0f0f0... */ pngout_init(pdb->nx, pdb->ny + 1); PNGINIT = 1; } else { memcpy(memheap + heapcount, buf + dataoff, datalen); MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount), datalen); heapcount += datalen; } unotice("records_per_block %d blocks_per_record %d nx %d ny %d", pdh->records_per_block, pdh->blocks_per_record, pdb->nx, pdb->ny); unotice("source %d sector %d channel %d", pdb->source, pdb->sector, pdb->channel); unotice("nrec %d recsize %d date %02d%02d%02d %02d%02d " "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month, pdb->day, pdb->hour, pdb->minute, pdb->second, pdb->sechunds); unotice("pshname %s", psh->pname); } deflen = 0; #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ if(buff_hdr->proc_blkno != 0){ /*acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;*/ acq_tbl->proc_prod_bytes_read += datalen; } } #endif } else { /* If the product already has a FOS trailer, don't add * another....this will match what pqing(SDI) sees */ if ((prod.nfrag != 0) && (prod.tail != NULL)) { if ((pfrag->fragnum != prod.tail->fragnum + 1) || (pfrag->seqno != prod.seqno)) { uerror("Missing fragment in sequence, last %d/%d this " "%d/%d\0", prod.tail->fragnum, prod.seqno, pfrag->fragnum, pfrag->seqno); #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ acq_tbl->proc_acqtab_prodseq_errs++; if(ulogIsDebug()) udebug("do_prod_mismatch() proc_base_prod_seqno_last = %d \n", acq_tbl->proc_base_prod_seqno_last); do_prod_mismatch(acq_tbl,buff_hdr); acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; } #endif ds_free(); prod.head = NULL; prod.tail = NULL; continue; } } if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) { char testme[4]; while (datalen > 4) { memcpy(testme, buf + (dataoff + datalen - 4), 4); if (memcmp(testme, FOS_TRAILER, 4) == 0) { datalen -= 4; if (ulogIsDebug()) udebug("removing FOS trailer from %s", PROD_NAME); } else { break; } } } if (heapcount + datalen > heapsize) { /* * this above wasn't big enough heapsize = * prodalloc(psh->frags,4000+15,&memheap); */ uerror("Error in heapsize %d product size %d [%d %d], Punt!\0", heapsize, (heapcount + datalen), heapcount, datalen); #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ /* Update Statistics */ acq_tbl->proc_tot_prods_lost_errs++; /* Abort entry and request retransmission */ prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR); generate_retrans_rqst(acq_tbl, prod.seqno, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR); if(acq_tbl->proc_orig_prod_seqno_last != 0){ strcpy(log_buff, "RETRANS"); } log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last, buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno, buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read, acq_tbl->proc_prod_start_time); acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; } #endif continue; } memcpy(memheap + heapcount, buf + dataoff, datalen); deflen = datalen; MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount), deflen); #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ if(buff_hdr->proc_blkno != 0){ /*acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;*/ acq_tbl->proc_prod_bytes_read += datalen; } } #endif } pfrag->recsiz = deflen; heapcount += deflen; if (prod.head == NULL) { prod.head = pfrag; prod.tail = pfrag; } else { prod.tail->next = pfrag; prod.tail = pfrag; } #ifdef RETRANS_SUPPORT if(((prod.nfrag == 0) || (prod.nfrag >= (pfrag->fragnum +1))) && (save_prod == 0)){ if(ulogIsVerbose()) uinfo("Do not save prod [seqno=%ld] as its retrans dup fragnum/total fragments =[%d of %d] save_prod=[%d] \n", prod.seqno,pfrag->fragnum,prod.nfrag,save_prod); ds_free (); prod.head = NULL; prod.tail = NULL; PNGINIT = 0; }else{ #endif if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) { if (GOES == 1) { if (PNGINIT == 1) { pngout_end(); heapcount = png_get_prodlen(); } else { if (ulogIsDebug()) udebug("GOES product already compressed %d", heapcount); } } if (ulogIsVerbose()) uinfo("we should have a complete product %ld %ld/%ld %ld /heap " "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum, (long) heapcount); if ((NWSTG == 1) && (heapcount > 4)) { cnt = 4; /* number of bytes to add for TRAILER */ /* * Do a DDPLUS vs HDS check for NWSTG channel only */ if (sbn->datastream == 5) { /* nwstg channel */ switch (psh->pcat) { case 1: case 7: /* Do a quick check for non-ascii text products */ if (!prod_isascii(PROD_NAME, memheap, heapcount)) psh->pcat += 100; /* call these HDS */ break; } } if (cnt > 0) { memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt); MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount), cnt); heapcount += cnt; } } #ifdef RETRANS_SUPPORT if((retrans_xmit_enable == OPTION_ENABLE) && (acq_tbl->read_distrib_enable & READ_CTL_DISCARD)){ num_prod_discards++; /* Set discard_prod to 1; Otherwise already stored prod may be requested for retransmit */ discard_prod=1; if(ulogIsVerbose()) uinfo("No of products discarded = %ld prod.seqno=%ld \n ",num_prod_discards,prod.seqno); prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR); acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno -1 ; ds_free (); prod.head = NULL; prod.tail = NULL; PNGINIT = 0; }else{ /* Do not insert prod into queue if its a duplicate product */ if(save_prod != 0) #endif process_prod(prod, PROD_NAME, memheap, heapcount, md5ctxp, productMaker->ldmProdQueue, psh, sbn); #ifdef RETRANS_SUPPORT /* Update acq table with last processed seqno -Begin */ if(retrans_xmit_enable == OPTION_ENABLE){ acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; uinfo(" prod with seqno processed = %ld\n",acq_tbl->proc_base_prod_seqno_last); } /* Update acq table with last processed seqno -End */ #endif ds_free(); prod.head = NULL; prod.tail = NULL; PNGINIT = 0; (void)pthread_mutex_lock(&productMaker->mutex); productMaker->nprods++; (void)pthread_mutex_unlock(&productMaker->mutex); #ifdef RETRANS_SUPPORT } #endif } else { if (ulogIsDebug()) udebug("processing record %ld [%ld %ld]", prod.seqno, prod.nfrag, pfrag->fragnum); if ((pdh->transtype & 4) > 0) { uerror("Hmmm....should call completed product %ld [%ld %ld]", prod.seqno, prod.nfrag, pfrag->fragnum); } } #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ if(!(acq_tbl->read_distrib_enable & READ_CTL_DISCARD)) if(!discard_prod){ acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno; discard_prod = 0; } } #endif #ifdef RETRANS_SUPPORT } save_prod = 1; #endif IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize); if (ulogIsDebug()) udebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen, deflen); #ifdef RETRANS_SUPPORT if(retrans_xmit_enable == OPTION_ENABLE){ total_prods_retrans_rcvd = acq_tbl->proc_tot_prods_retrans_rcvd; /* prods retrans rcvd by proc */ total_prods_retrans_rcvd_lost = acq_tbl->proc_tot_prods_retrans_rcvd_lost; /* prods retrans rcvd lost */ total_prods_retrans_rcvd_notlost = acq_tbl->proc_tot_prods_retrans_rcvd_notlost; /* prods rcvd not lost */ total_prods_retrans_rqstd = acq_tbl->proc_tot_prods_retrans_rqstd; /* prods retrans requested */ total_prods_handled = acq_tbl->proc_tot_prods_handled; /* prods retrans requested */ total_prods_lost_err = acq_tbl->proc_tot_prods_lost_errs; /* prods retrans requested */ total_frame_cnt = acq_tbl->read_tot_buff_read; total_frame_err = acq_tbl->read_frame_tot_lost_errs; proc_orig_prod_seqno_last_save = acq_tbl->proc_orig_prod_seqno_last; } #endif } if (NULL != memheap) free(memheap); productMaker->status = status; return NULL; }
/** * Reads NOAAPORT data from a shared-memory FIFO or a file, creates LDM * data-products, and inserts the data-products into an LDM product-queue. * * Usage: * * readnoaaport [-nvx] [-q <em>queue</em>] [-u <em>n</em>] [-m mcastAddr] [path]\n * * Where: * <dl> * <dt>-l <em>log</em></dt> * <dd>Log to \e log. if \e log is "-", then logging occurs to the * standard error stream; otherwise, \e log is the pathname of a file to * which logging will occur. If not specified, then log messages will go * to the system logging daemon. </dd> * * <dt>-m <em>mcastAddr</em></dt> * <dd>Use the shared-memory FIFO associated with the UDP * multicast address \e mcastAddr.</dd> * * <dt>-n</dt> * <dd>Log messages of level NOTICE and higher priority.</dd> * * <dt>-q <em>queue</em></dt> * <dd>Use \e queue as the pathname of the LDM product-queue. The default * is to use the default LDM pathname of the product-queue.</dd> * * <dt>-u <em>n</em></dt> * <dd>If logging is to the system logging daemon, then use facility * <b>local</b><em>n</em>. The default is to use the LDM facility. * * <dt>-v</dt> * <dd>Log messages of level INFO and higher priority. Each data-product * will generate a log message.</dd> * * <dt>-x</dt> * <dd>Log messages of level DEBUG and higher priority.</dd> * * <dt><em>path</em></dt> * <dd>Pathname of the file from which to read data. The default is to use * a shared-memory FIFO.</dd> * </dl> * * @retval 0 if successful. * @retval 1 if an error occurred. At least one error-message is logged. */ int main( const int argc, char* const argv[]) { #ifdef HAVE_GET_QUEUE_PATH const char* pqfname = getQueuePath(); #else const char* pqfname = DEFAULT_QUEUE; #endif int fd; char* prodmmap; char* memheap = NULL; size_t heapsize; size_t heapcount; unsigned char b1; int cnt, dataoff, datalen, deflen; int nscan; long IOFF; int NWSTG, GOES, PNGINIT = 0, PROD_COMPRESSED; long last_sbn_seqno = (-1); char PROD_NAME[1024]; int status; prodstore prod; sbn_struct* sbn; pdh_struct* pdh; psh_struct* psh; ccb_struct* ccb; pdb_struct* pdb; datastore* pfrag; extern int optind; extern int opterr; extern char* optarg; int ch; int logmask = LOG_MASK(LOG_ERR); const char* logfname = NULL; /* use system logging daemon */ unsigned logOptions = LOG_CONS | LOG_PID; unsigned logFacility = LOG_LDM; /* use default LDM facility */ const char* const progName = ubasename(argv[0]); MD5_CTX* md5ctxp = NULL; /*unsigned char *compr; long comprLen = 10000 * sizeof (int);*/ int pid_channel = -1; /*compr = (unsigned char *) calloc (comprLen, 1);*/ /* Initialize the logger. */ (void)setulogmask(logmask); (void)openulog(progName, logOptions, logFacility, logfname); opterr = 1; while ((ch = getopt(argc, argv, "nvxl:q:u:m:")) != EOF) { switch (ch) { case 'v': logmask |= LOG_MASK(LOG_INFO); (void)setulogmask(logmask); break; case 'x': logmask |= LOG_MASK(LOG_DEBUG); (void)setulogmask(logmask); break; case 'n': logmask |= LOG_MASK(LOG_NOTICE); (void)setulogmask(logmask); break; case 'l': if (optarg[0] == '-' && optarg[1] != 0) { nplError("logfile \"%s\" ??\n", optarg); usage(argv[0]); } /* else */ logfname = optarg; (void)openulog(progName, logOptions, logFacility, logfname); break; case 'q': pqfname = optarg; break; case 'u': { int i = atoi(optarg); if (0 <= i && 7 >= i) { static int logFacilities[] = {LOG_LOCAL0, LOG_LOCAL1, LOG_LOCAL2, LOG_LOCAL3, LOG_LOCAL4, LOG_LOCAL5, LOG_LOCAL6, LOG_LOCAL7}; logFacility = logFacilities[i]; (void)openulog(progName, logOptions, logFacility, logfname); } break; } case 'm': sscanf(optarg, "%*d.%*d.%*d.%d", &pid_channel); if ((pid_channel < 1) || (pid_channel > MAX_DVBS_PID)) { pid_channel = -1; } else { shm = shmfifo_new(); cnt = 0; while (((status = shmfifo_shm_from_key(shm, s_port[pid_channel - 1])) == -3) && (cnt < 30)) { nplInfo("Trying to get shared-memory FIFO"); cnt++; sleep(1); } if (0 != status) { nplError("Couldn't get shared-memory FIFO. " "Check associated dvbs_multicast(1) process."); shmfifo_free(shm); shm = NULL; } else { nplInfo("Got shared-memory FIFO"); } } break; case '?': usage(argv[0]); break; } } if (argc - optind < 0) usage(argv[0]); nplNotice("Starting Up %s", PACKAGE_VERSION); fd = ((argc - optind) == 0) ? fileno(stdin) : open(argv[optind], O_RDONLY, 0); if ((!shm) && (fd == -1)) { nplError("could not open input file"); exit(0); } /* * Set up signal handlers */ set_sigactions(); /* * Register atexit routine */ if (atexit(cleanup) != 0) { nplSerror("atexit"); exit(-1); } sbn = (sbn_struct*)malloc(sizeof(sbn_struct)); pdh = (pdh_struct*)malloc(sizeof(pdh_struct)); psh = (psh_struct*)malloc(sizeof(psh_struct)); ccb = (ccb_struct*)malloc(sizeof(ccb_struct)); pdb = (pdb_struct*)malloc(sizeof(pdb_struct)); prodmmap = (char*)malloc(10000); if (prodmmap == NULL) { nplError("could not allocate read buffer"); exit(-1); } md5ctxp = new_MD5_CTX(); prod.head = NULL; prod.tail = NULL; if (lpqGet(pqfname, &ldmProdQueue) != 0) { NPL_ADD1("Couldn't open LDM product-queue \"%s\"", pqfname); exit(1); } while (DONE == 0) { /* See if any stats need to be logged */ if (logstats) { logstats = 0; dump_stats(); } /* Look for first byte == 255 and a valid SBN checksum */ if ((status = bufread(fd, prodmmap, 1)) != 0) { if (-3 == status) break; abort(); } if ((b1 = (unsigned char)prodmmap[0]) != 255) { if (ulogIsVerbose()) nplInfo("trying to resync %u", b1); if (ulogIsDebug()) nplDebug("bufread loop"); continue; } if (bufread(fd, prodmmap + 1, 15) != 0) { if (ulogIsDebug()) nplDebug("couldn't read 16 bytes for sbn"); continue; } while ((status = readsbn(prodmmap, sbn)) != 0) { if (ulogIsDebug()) nplDebug("Not SBN start"); IOFF = 1; while ((IOFF < 16) && ((b1 = (unsigned char) prodmmap[IOFF]) != 255)) IOFF++; if (IOFF > 15) { break; } else { for (ch = IOFF; ch < 16; ch++) prodmmap[ch - IOFF] = prodmmap[ch]; if (bufread(fd, prodmmap + 16 - IOFF, IOFF) != 0) { if (ulogIsDebug()) nplDebug("Couldn't read bytes for SBN, resync"); break; } } } if (status != 0) { if (ulogIsDebug()) nplDebug("SBN status continue"); continue; } IOFF = 0; if (bufread(fd, prodmmap + 16, 16) != 0) { if (ulogIsDebug()) nplDebug("error reading Product Definition Header"); continue; } if (ulogIsDebug()) nplDebug("***********************************************"); if (last_sbn_seqno != -1) { if (sbn->seqno != last_sbn_seqno + 1) { nplNotice("Gap in SBN sequence number %ld to %ld [skipped %ld]", last_sbn_seqno, sbn->seqno, sbn->seqno - last_sbn_seqno - 1); if ( sbn->seqno > last_sbn_seqno ) nmissed = nmissed + (unsigned long)(sbn->seqno - last_sbn_seqno - 1); } } last_sbn_seqno = sbn->seqno; if (ulogIsVerbose()) nplInfo("SBN seqnumber %ld", sbn->seqno); if (ulogIsVerbose()) nplInfo("SBN datastream %d command %d", sbn->datastream, sbn->command); if (ulogIsDebug()) nplDebug("SBN version %d length offset %d", sbn->version, sbn->len); if (((sbn->command != 3) && (sbn->command != 5)) || (sbn->version != 1)) { nplError("Unknown sbn command/version %d PUNT", sbn->command); continue; } switch (sbn->datastream) { case 7: /* test */ case 6: /* was reserved...now nwstg2 */ case 5: NWSTG = 1; GOES = 0; break; case 1: case 2: case 4: NWSTG = 0; GOES = 1; break; default: nplError("Unknown NOAAport channel %d PUNT", sbn->datastream); continue; } /* End of SBN version low 4 bits */ if (readpdh(prodmmap + IOFF + sbn->len, pdh) == -1) { nplError("problem with pdh, PUNT"); continue; } if (pdh->len > 16) { bufread(fd, prodmmap + sbn->len + 16, pdh->len - 16); } if (ulogIsDebug()) nplDebug("Product definition header version %d pdhlen %d", pdh->version, pdh->len); if (pdh->version != 1) { nplError("Error: PDH transfer type %u, PUNT", pdh->transtype); continue; } else if (ulogIsDebug()) { nplDebug("PDH transfer type %u", pdh->transtype); } if ((pdh->transtype & 8) > 0) nplError("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 32) > 0) nplError("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 16) > 0) { PROD_COMPRESSED = 1; if (ulogIsDebug()) nplDebug("Product transfer flag compressed %u", pdh->transtype); } else { PROD_COMPRESSED = 0; } if (ulogIsDebug()) nplDebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen, pdh->pshlen); if (ulogIsDebug()) nplDebug("blocks per record %ld records per block %ld\n", pdh->blocks_per_record, pdh->records_per_block); if (ulogIsDebug()) nplDebug("product seqnumber %ld block number %ld data block size " "%ld", pdh->seqno, pdh->dbno, pdh->dbsize); /* Stop here if no psh */ if ((pdh->pshlen == 0) && (pdh->transtype == 0)) { IOFF = IOFF + sbn->len + pdh->len; continue; } if (pdh->pshlen != 0) { if (bufread(fd, prodmmap + sbn->len + pdh->len, pdh->pshlen) != 0) { nplError("problem reading psh"); continue; } else { if (ulogIsDebug()) nplDebug("read psh %d", pdh->pshlen); } /* Timing block */ if (sbn->command == 5) { if (ulogIsDebug()) nplDebug("Timing block recieved %ld %ld\0", psh->olen, pdh->len); /* * Don't step on our psh of a product struct of prod in * progress. */ continue; } if (readpsh(prodmmap + IOFF + sbn->len + pdh->len, psh) == -1) { nplError("problem with readpsh"); continue; } if (psh->olen != pdh->pshlen) { nplError("ERROR in calculation of psh len %ld %ld", psh->olen, pdh->len); continue; } if (ulogIsDebug()) nplDebug("len %ld", psh->olen); if (ulogIsDebug()) nplDebug("product header flag %d, version %d", psh->hflag, psh->version); if (ulogIsDebug()) nplDebug("prodspecific data length %ld", psh->psdl); if (ulogIsDebug()) nplDebug("bytes per record %ld", psh->bytes_per_record); if (ulogIsDebug()) nplDebug("Fragments = %ld category %d ptype %d code %d", psh->frags, psh->pcat, psh->ptype, psh->pcode); if (psh->frags < 0) nplError("check psh->frags %d", psh->frags); if (psh->origrunid != 0) nplError("original runid %d", psh->origrunid); if (ulogIsDebug()) nplDebug("next header offset %ld", psh->nhoff); if (ulogIsDebug()) nplDebug("original seq number %ld", psh->seqno); if (ulogIsDebug()) nplDebug("receive time %ld", psh->rectime); if (ulogIsDebug()) nplDebug("transmit time %ld", psh->transtime); if (ulogIsDebug()) nplDebug("run ID %ld", psh->runid); if (ulogIsDebug()) nplDebug("original run id %ld", psh->origrunid); if (prod.head != NULL) { nplError("OOPS, start of new product [%ld ] with unfinished " "product %ld", pdh->seqno, prod.seqno); ds_free(); prod.head = NULL; prod.tail = NULL; if (PNGINIT != 0) { pngout_end(); PNGINIT = 0; } nplError("Product definition header version %d pdhlen %d", pdh->version, pdh->len); nplError("PDH transfer type %u", pdh->transtype); if ((pdh->transtype & 8) > 0) nplError("Product transfer flag error %u", pdh->transtype); if ((pdh->transtype & 32) > 0) nplError("Product transfer flag error %u", pdh->transtype); nplError("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen, pdh->pshlen); nplError("blocks per record %ld records per block %ld", pdh->blocks_per_record, pdh->records_per_block); nplError("product seqnumber %ld block number %ld data block " "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize); nplError("product header flag %d", psh->hflag); nplError("prodspecific data length %ld", psh->psdl); nplError("bytes per record %ld", psh->bytes_per_record); nplError("Fragments = %ld category %d", psh->frags, psh->pcat); if (psh->frags < 0) nplError("check psh->frags %d", psh->frags); if (psh->origrunid != 0) nplError("original runid %d", psh->origrunid); nplError("next header offset %ld", psh->nhoff); nplError("original seq number %ld", psh->seqno); nplError("receive time %ld", psh->rectime); nplError("transmit time %ld", psh->transtime); nplError("run ID %ld", psh->runid); nplError("original run id %ld", psh->origrunid); } prod.seqno = pdh->seqno; prod.nfrag = psh->frags; ds_init(prod.nfrag); /* NWSTG CCB = dataoff, WMO = dataoff + 24 */ if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen, pdh->dbsize) != 0) { nplError("problem reading datablock"); continue; } if (sbn->datastream == 4) { if (psh->pcat != 3) { GOES = 0; NWSTG = 1; } } heapcount = 0; MD5Init(md5ctxp); if (GOES == 1) { if (readpdb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen, psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) { nplError("Error reading pdb, punt"); continue; } memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME)); if (ulogIsDebug()) nplDebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len, pdh->pshlen, sbn->len + pdh->len + pdh->pshlen, pdb->len); /* Data starts at first block after pdb */ ccb->len = 0; heapsize = prodalloc(psh->frags, 5152, &memheap); } if (NWSTG == 1) { memset(psh->pname, 0, sizeof(psh->pname)); if (readccb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen, ccb, psh, pdh->dbsize) == -1) nplError("Error reading ccb, using default name"); if (ulogIsDebug()) nplDebug("look at ccb start %d %d", ccb->b1, ccb->len); /* cnt = 0; memset(psh->pname,0,sizeof(psh->pname)); while ((b1 = (unsigned char)prodmmap[ IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len + cnt]) >= 32) { psh->pname[cnt] = prodmmap[ IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len + cnt]; cnt++; } if(cnt > 0) */ if (ulogIsVerbose()) nplInfo("%s", psh->pname); memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME)); heapsize = prodalloc(psh->frags, 4000 + 15, &memheap); /* * We will only compute md5 checksum on the data, 11 FOS * characters at start */ /* * sprintf(memheap,"\001\015\015\012%04d\015\015\012", * ((int)pdh->seqno)%10000); */ sprintf(memheap, "\001\015\015\012%03d\040\015\015\012", ((int) pdh->seqno) % 1000); heapcount += 11; if (psh->metaoff > 0) psh->metaoff = psh->metaoff + 11; } } else { /* If a continuation record...don't let psh->pcat get missed */ if ((sbn->datastream == 4) && (psh->pcat != 3)) { GOES = 0; NWSTG = 1; } ccb->len = 0; if (ulogIsDebug()) nplDebug("continuation record"); if ((pdh->transtype & 4) > 0) { psh->frags = 0; } if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen, pdh->dbsize) != 0) { nplError("problem reading datablock (cont)"); continue; } if (prod.head == NULL) { if (ulogIsVerbose()) nplInfo("found data block before header, " "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno); continue; } } /* Get the data */ dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len; datalen = pdh->dbsize - ccb->len; if (ulogIsDebug()) nplDebug("look at datalen %d", datalen); pfrag = ds_alloc(); pfrag->seqno = pdh->seqno; pfrag->fragnum = pdh->dbno; pfrag->recsiz = datalen; pfrag->offset = heapcount; pfrag->next = NULL; /*memcpy(memheap+heapcount,prodmmap+dataoff,datalen); MD5Update(md5ctxp, (unsigned char *)(memheap+heapcount), datalen); test_deflate(compr,comprLen,(unsigned char *)(memheap+heapcount), datalen);*/ if (GOES == 1) { if (pfrag->fragnum > 0) { if ((pfrag->fragnum != prod.tail->fragnum + 1) || (pfrag->seqno != prod.seqno)) { nplError("Missing GOES fragment in sequence, " "last %d/%d this %d/%d\0", prod.tail->fragnum, prod.seqno, pfrag->fragnum, pfrag->seqno); ds_free(); prod.head = NULL; prod.tail = NULL; continue; } if ((PNGINIT != 1) && (!PROD_COMPRESSED)) { nplError("failed pnginit %d %d %s", sbn->datastream, psh->pcat, PROD_NAME); continue; } if (pdh->records_per_block < 1) { nplError("records_per_block %d blocks_per_record %d " "nx %d ny %d", pdh->records_per_block, pdh->blocks_per_record, pdb->nx, pdb->ny); nplError("source %d sector %d channel %d", pdb->source, pdb->sector, pdb->channel); nplError("nrec %d recsize %d date %02d%02d%02d %02d%02d " "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month, pdb->day, pdb->hour, pdb->minute, pdb->second, pdb->sechunds); nplError("pshname %s", psh->pname); } if (!PROD_COMPRESSED) { for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) { if (ulogIsDebug()) nplDebug("png write nscan %d", nscan); if (nscan >= pdh->records_per_block) { nplError("nscan exceeding records per block %d [%d " "%d %d]", pdh->records_per_block, nscan, pdb->nx, pdh->dbsize); } else { pngwrite(prodmmap + dataoff + (nscan * pdb->nx)); } } } else { memcpy(memheap + heapcount, prodmmap + dataoff, datalen); MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount), datalen); heapcount += datalen; } } else { if (!PROD_COMPRESSED) { png_set_memheap(memheap, md5ctxp); png_header(prodmmap + dataoff, datalen); /* * Add 1 to number of scanlines, image ends with * f0f0f0f0... */ pngout_init(pdb->nx, pdb->ny + 1); PNGINIT = 1; } else { memcpy(memheap + heapcount, prodmmap + dataoff, datalen); MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount), datalen); heapcount += datalen; } nplNotice("records_per_block %d blocks_per_record %d nx %d ny %d", pdh->records_per_block, pdh->blocks_per_record, pdb->nx, pdb->ny); nplNotice("source %d sector %d channel %d", pdb->source, pdb->sector, pdb->channel); nplNotice("nrec %d recsize %d date %02d%02d%02d %02d%02d " "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month, pdb->day, pdb->hour, pdb->minute, pdb->second, pdb->sechunds); nplNotice("pshname %s", psh->pname); } deflen = 0; } else { /* * test_deflate(memheap+heapcount,heapsize-heapcount,(unsigned char * *)(prodmmap+dataoff),datalen,&deflen); */ /* If the product already has a FOS trailer, don't add * another....this will match what pqing(SDI) sees */ if ((prod.nfrag != 0) && (prod.tail != NULL)) { if ((pfrag->fragnum != prod.tail->fragnum + 1) || (pfrag->seqno != prod.seqno)) { nplError("Missing fragment in sequence, last %d/%d this " "%d/%d\0", prod.tail->fragnum, prod.seqno, pfrag->fragnum, pfrag->seqno); ds_free(); prod.head = NULL; prod.tail = NULL; continue; } } if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) { char testme[4]; while (datalen > 4) { memcpy(testme, prodmmap + (dataoff + datalen - 4), 4); if (memcmp(testme, FOS_TRAILER, 4) == 0) { datalen -= 4; if (ulogIsDebug()) nplDebug("removing FOS trailer from %s", PROD_NAME); } else { break; } } } if (heapcount + datalen > heapsize) { /* * this above wasn't big enough heapsize = * prodalloc(psh->frags,4000+15,&memheap); */ nplError("Error in heapsize %d product size %d [%d %d], Punt!\0", heapsize, (heapcount + datalen), heapcount, datalen); continue; } memcpy(memheap + heapcount, prodmmap + dataoff, datalen); deflen = datalen; MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount), deflen); } pfrag->recsiz = deflen; /*heapcount += datalen;*/ heapcount += deflen; if (prod.head == NULL) { prod.head = pfrag; prod.tail = pfrag; } else { prod.tail->next = pfrag; prod.tail = pfrag; } if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) { if (GOES == 1) { if (PNGINIT == 1) { pngout_end(); heapcount = png_get_prodlen(); } else { if (ulogIsDebug()) nplDebug("GOES product already compressed %d", heapcount); } } if (ulogIsVerbose()) nplInfo("we should have a complete product %ld %ld/%ld %ld /heap " "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum, (long) heapcount); if ((NWSTG == 1) && (heapcount > 4)) { cnt = 4; /* number of bytes to add for TRAILER */ /* * Do a DDPLUS vs HDS check for NWSTG channel only */ if (sbn->datastream == 5) { /* nwstg channel */ switch (psh->pcat) { case 1: case 7: /* Do a quick check for non-ascii text products */ if (!prod_isascii(PROD_NAME, memheap, heapcount)) psh->pcat += 100; /* call these HDS */ /* else { ** call these DDPLUS ** if (memheap[heapcount-1] == FOS_TRAILER[3]) ** ETX check ** cnt = 0; ** no need to add extra ETX pqing doesn't see it ** } */ break; } } if (cnt > 0) { memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt); MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount), cnt); heapcount += cnt; } } process_prod(prod, PROD_NAME, memheap, heapcount, md5ctxp, ldmProdQueue, psh, sbn); ds_free(); prod.head = NULL; prod.tail = NULL; PNGINIT = 0; } else { if (ulogIsDebug()) nplDebug("processing record %ld [%ld %ld]", prod.seqno, prod.nfrag, pfrag->fragnum); if ((pdh->transtype & 4) > 0) { nplError("Hmmm....should call completed product %ld [%ld %ld]", prod.seqno, prod.nfrag, pfrag->fragnum); } } IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize); if (ulogIsDebug()) nplDebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen, deflen); } if (fd != -1) (void)close(fd); exit(0); }
/* * Reads data from a file descriptor. * * Arguments: * fd The file descriptor from which to read data. * buf Pointer to the buffer into which to put the data. * want The amount of data to read in bytes. * Returns: * 0 Success. * -2 I/O error. Error-message logged. * -3 End-of-file. */ static int _fd_bufread( const int fd, char* const buf, const int bsiz) { int width; int ready; struct timeval timeo; size_t bread = 0; static int TV_SEC = 30; width = fd + 1; while (bread < bsiz) { timeo.tv_sec = TV_SEC; timeo.tv_usec = 0; FD_ZERO(&readfds); FD_ZERO(&exceptfds); FD_SET(fd, &readfds); FD_SET(fd, &exceptfds); ready = select(width, &readfds, 0, &exceptfds, &timeo); /* timeo may be modified, don't rely on value now */ if (ready < 0) { if (errno != EINTR) nplSerror("select"); else nplNotice("select received interupt\0"); errno = 0; continue; } if (ready == 0) { idle += TV_SEC; if (idle > 600) { if (ulogIsVerbose()) nplInfo("Idle for 600 seconds"); idle = 0; } else if (ulogIsDebug()) { nplDebug("Idle for %d seconds", idle); } continue; /* was return(-1) */ } if (FD_ISSET(fd, &readfds) || FD_ISSET(fd, &exceptfds)) { size_t nread; idle = 0; nread = read(fd, buf + bread, (size_t) bsiz - bread); bread = bread + nread; if (nread == -1) { nplSerror("_fd_bufread(): read() failure"); return -2; } if (nread == 0) { if (ulogIsVerbose()) nplInfo("End of Input"); DONE = 1; return -3; } if (bread < (size_t) bsiz) { FD_CLR(fd, &readfds); FD_CLR(fd, &exceptfds); } else { return 0; } } else { nplError("_fd_bufread(): select() returned %d but fd not set", ready); idle += TV_SEC; return -2; } } return 0; }
int main( int ac, char *av[] ) { char *progname = av[0]; char *logfname; int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; logfname = "-"; /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv("LDMPQFNAME"); if(ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) | LOG_MASK(LOG_NOTICE)); opterr = 1; while ((ch = getopt(ac, av, "ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': logmask |= LOG_MASK(LOG_INFO); break; case 'x': logmask |= LOG_MASK(LOG_DEBUG); break; case 'l': logfname = optarg; break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case '?': usage(progname); break; } ac -= optind; av += optind ; if(ac < 1) usage(progname); (void) setulogmask(logmask); } /* * Set up error logging */ (void) openulog(ubasename(progname), LOG_NOTIME, LOG_LDM, logfname); /* * register exit handler */ if(atexit(cleanup) != 0) { serror("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strcpy(myname, ghostname()); /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { uerror("The product-queue \"%s\" is inconsistent\n", pqfname); } else { uerror("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { serror("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { serror("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { serror("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; #ifndef NO_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == NULL) { serror("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { serror("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else /*!NO_MMAP*/ status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { serror("fd_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { serror("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } index = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &index); if(status != ENOERR) { serror("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { serror("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, index); index = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())) ; break; case PQUEUE_DUP: uerror("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); break; case ENOMEM: uerror("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: uerror("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into "index" region */ if (status != ENOERR) { (void)pqe_discard(pq, index); index = PQE_NONE; } } /* "index" region allocated */ #endif /*!NO_MMAP*/ (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
/* * Send a product from file-descriptor to clnt using LDM-6 protocol. */ static void send_product_6(CLIENT *clnt, int fd, prod_info *infop) { unsigned size = infop->sz; if (size <= max_hereis) { /* * The file is small enough to be sent in a single HEREIS message. */ void* buf = (char*)malloc(size); udebug("Sending file via HEREIS"); if (NULL == buf) { serror("Couldn't allocate %u bytes for product data", size); } else { ssize_t nread = read(fd, buf, size); if(nread != size) { serror("Couldn't read %u bytes of data", size); } else { product product; product.info = *infop; product.data = buf; if (NULL == hereis_6(&product, clnt)) { uerror("%s: HEREIS_6 failure: %s", remote, clnt_errmsg(clnt)); } } free(buf); } } else { /* * The file is so large that it must be sent via COMINGSOON/BLKDATA * messages. */ comingsoon_reply_t* reply; comingsoon_args soonArg; udebug("Sending file via COMINGSOON/BLKDATA"); soonArg.infop = infop; soonArg.pktsz = size; reply = comingsoon_6(&soonArg, clnt); if (NULL == reply) { uerror("%s: COMINGSOON_6 failure: %s", remote, clnt_errmsg(clnt)); } else { if (DONT_SEND == *reply) { if (ulogIsVerbose() || ulogIsDebug()) uinfo("Downstream LDM says don't send: %s", s_prod_info(NULL, 0, infop, ulogIsDebug())); } else if (0 != *reply) { uwarn("Unexpected reply (%s) from downstream LDM: %s", s_prod_info(NULL, 0, infop, ulogIsDebug())); } else { void* buf = (char*)malloc(size); if (NULL == buf) { serror("Couldn't allocate %u bytes for product data", size); } else { ssize_t nread = read(fd, buf, size); if(nread != size) { serror("Couldn't read %u bytes of data", size); } else { datapkt packet; packet.signaturep = (signaturet*)&infop->signature; packet.pktnum = 0; packet.data.dbuf_len = size; packet.data.dbuf_val = buf; if (NULL == blkdata_6(&packet, clnt)) { uerror("%s: BLKDATA_6 failure: %s", remote, clnt_errmsg(clnt)); } } free(buf); } } } } }
/* * Arguments: * arrival Data-product creation-time. IGNORED. The creation-time * will be set by this function according to the system clock * just prior to inserting the data-product into the product- * queue. * seqno Sequence number. * ident Product-identifier. * len Size of data-portion of data-product in bytes. * buf Pointer to data-portion of data-product. */ void toClients(timestampt arrival, unsigned seqno, const char *ident, unsigned len, const char *buf) { static struct product prod; int status; char *result; MD5Init(md5ctxp); /* * If user has not disabled skipLeadingCtlString with -5 option, * and the product contains the correct leading control string for * a wmo product, then skip that control string in calculating the * checksum. */ if (skipLeadingCtlString && ((result = wmo_prod(buf)) != 0)) { MD5Update(md5ctxp, (const unsigned char *)result, len-(result-buf)); #if DEBUG uinfo("WMO prod: Skipping %d chars\n", result-buf); #endif } else /* calculate checksum on entire product */ { MD5Update(md5ctxp, (const unsigned char *)buf, len); #if DEBUG uinfo("not a WMO Prod\n"); #endif } MD5Final((unsigned char*)prod.info.signature, md5ctxp); prod.info.origin = myname; prod.info.feedtype = feedtype; prod.info.seqno = seqno; prod.info.ident = (char *)ident; /* cast away const */ prod.info.sz = len; prod.data = (void *)buf; /* cast away const */ if(((strncmp(prod.info.ident,"SDUS2",5) == 0) || (strncmp(prod.info.ident,"SDUS3",5) == 0) || (strncmp(prod.info.ident,"SDUS5",5) == 0) || (strncmp(prod.info.ident,"SDUS7",5) == 0)) && (useNex == 1)) { prod.info.feedtype = NEXRAD; } if(ulogIsVerbose()) uinfo("%s", s_prod_info(NULL, 0, &prod.info, ulogIsDebug())); if(pq == NULL) /* if we are "feedtest", do nothing else */ return; set_timestamp(&prod.info.arrival); status = pq_insert(pq, &prod); if(status == ENOERR) return; /* Normal return */ /* else */ if(status == PQUEUE_DUP) { ndups++; uinfo("Product already in queue"); return; } /* else, error */ uerror("pq_insert: %s\n", status > 0 ? strerror(status) : "Internal error"); exit(1); /* ??? */ }