int do_prod_lost( BUFF_HDR *buff_hdr, ACQ_TABLE *acq_tbl) { long prod_errors; if(acq_tbl->proc_base_prod_seqno_last == 0){ prod_errors = 0; }else{ prod_errors = (buff_hdr->proc_prod_seqno - (acq_tbl->proc_base_prod_seqno_last + 1)); } if(prod_errors <= 0){ prod_errors = 0; }else{ /* Now generate retransmission request */ generate_retrans_rqst(acq_tbl,(acq_tbl->proc_base_prod_seqno_last + 1), (buff_hdr->proc_prod_seqno-1), RETRANS_RQST_CAUSE_RCV_ERR); } acq_tbl->proc_tot_prods_lost_errs += prod_errors; /* Need to log prod_errors */ if(acq_tbl->proc_base_prod_seqno_last == 0){ log_info_q("LOST=%ld total(%ld) %s prod(%ld) prod_seqno was RESET to 0 \n", prod_errors,acq_tbl->proc_tot_prods_lost_errs, GET_PROD_TYPE_NAME(buff_hdr->proc_prod_type),buff_hdr->proc_prod_seqno); }else{ log_info_q("LOST=%ld total(%ld) %s prod(%ld) expect prod(%ld)\n", prod_errors, acq_tbl->proc_tot_prods_lost_errs, GET_PROD_TYPE_NAME(buff_hdr->proc_prod_type),buff_hdr->proc_prod_seqno, acq_tbl->proc_base_prod_seqno_last + 1); } return (0); }
/* * The RPC dispatch routine for this program. * Registered as a callback by svc_register() below. * Note that only NULLPROC and NOTIFICATION rpc procs are * handled by this program. */ static void notifymeprog_5(struct svc_req *rqstp, SVCXPRT *transp) { prod_info notice; switch (rqstp->rq_proc) { case NULLPROC: (void)svc_sendreply(transp, (xdrproc_t)xdr_void, (caddr_t)NULL); return; case NOTIFICATION: (void) memset((char*)¬ice, 0, sizeof(notice)); if (!svc_getargs(transp, (xdrproc_t)xdr_prod_info, (caddr_t)¬ice)) { svcerr_decode(transp); return; } (void)exitIfDone(0); /* * Update the request filter with the timestamp * we just recieved. * N.B.: There can still be duplicates after * a reconnect. */ clss.from = notice.arrival; timestamp_incr(&clss.from); /* * your code here, example just logs it */ log_info_q("%s", s_prod_info(NULL, 0, ¬ice, log_is_enabled_debug)); if(!svc_sendreply(transp, (xdrproc_t)xdr_ldm_replyt, (caddr_t) &reply)) { svcerr_systemerr(transp); } (void)exitIfDone(0); if(!svc_freeargs(transp, xdr_prod_info, (caddr_t) ¬ice)) { log_error_q("unable to free arguments"); exit(1); } /* no break */ default: svcerr_noproc(transp); return; } }
/** * There is data available on the feed. Read it into the buffer * then deal with what we got. * * @param ifd [in] File-descriptor of the input data-feed * @retval 0 Success * @retval ENOMEM Out of memory * @retval ENODATA End of input data-feed */ int feedTheXbuf(const int ifd) { int status; size_t nn = 0; /* space available in buffer */ ptrdiff_t remaining = (ptrdiff_t)theBuf->bufsiz - (theBuf->get - theBuf->base); if (remaining <= CHUNKSIZE) { if (theBuf->bufsiz >= maxProductSize) { log_warning_q( "Data-product would exceed %lu bytes. Resetting input buffer.", maxProductSize); justify_xbuf(theBuf, 0); } log_info_q("Expanding input buffer size to %lu\n", (unsigned long)(2 * theBuf->bufsiz)); theBuf = expand_xbuf(theBuf, theBuf->bufsiz); if (theBuf == NULL) { status = errno == 0 ? ENOMEM : errno; log_add_syserr("expand_xbuf"); log_flush_error(); return status; } } status = (*read_feed)(ifd, (char *)theBuf->put, CHUNKSIZE, &nn); if(status != ENOERR) { log_add_errno(status, "read_feed"); log_flush_error(); return status; } /* else */ if(nn == 0) return ENODATA; /* end of file */ /* else */ /* usual case */ /* assert(nn > 0); */ theBuf->cnt += nn; theBuf->put += nn; return ENOERR; }
int main(int ac, char *av[]) { int pflags = PQ_NOCLOBBER; off_t initialsz = 0; size_t nproducts = 0; pqueue *pq = NULL; int errnum = 0; /* * initialize logger */ (void)log_init(av[0]); int ch; char *qopt = NULL; char *sopt = NULL; char *Sopt = NULL; extern char *optarg; extern int optind; const char* pqfname = getQueuePath(); while ((ch = getopt(ac, av, "xvcfq:s:S:l:")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'c': pflags &= ~PQ_NOCLOBBER; break; case 'f': pflags |= PQ_SPARSE; break; case 's': sopt = optarg; break; case 'S': Sopt = optarg; break; case 'q': qopt = optarg; break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': (void)log_set_destination(optarg); break; case '?': usage(av[0]); break; } if(ac - optind > 1) { if(sopt) usage(av[0]); sopt = av[ac - 2]; } if(ac - optind > 0) { if(qopt) usage(av[0]); qopt = av[ac - 1]; } if(qopt) { pqfname = qopt ; setQueuePath(qopt); } if (sopt) { char* cp; int exponent = 0; errno = 0; initialsz = strtol(sopt, &cp, 0); if (0 != errno) { initialsz = 0; /* trigger error below */ } else { switch (*cp) { case 0: break; case 'k': case 'K': exponent = 1; break; case 'm': case 'M': exponent = 2; break; case 'g': case 'G': exponent = 3; break; default: initialsz = 0; /* trigger error below */ break; } if (0 < initialsz) { int i; for (i = 0; i < exponent; i++) { initialsz *= 1000; if (0 >= initialsz) { fprintf(stderr, "Size \"%s\" too big\n", sopt); usage(av[0]); } } } } } if(initialsz <= 0) { if(sopt) fprintf(stderr, "Illegal size \"%s\"\n", sopt); else fprintf(stderr, "No size specified\n"); usage(av[0]); } if(Sopt != NULL) { nproducts = (size_t)atol(Sopt); if(nproducts == 0) { fprintf(stderr, "Illegal nproducts \"%s\"\n", Sopt); } } else { #define PQ_AVG_PRODUCT_SIZE 51000 // approximate mean size on 2014-08-21 /* For default number of product slots, use average product size estimate */ nproducts = initialsz/PQ_AVG_PRODUCT_SIZE; } log_info_q("Creating %s, %ld bytes, %ld products.\n", pqfname, (long)initialsz, (long)nproducts); errnum = pq_create(pqfname, 0666, pflags, 0, initialsz, nproducts, &pq); if(errnum) { fprintf(stderr, "%s: create \"%s\" failed: %s\n", av[0], pqfname, strerror(errnum)); exit(1); } (void)pq_close(pq); return(0); }
hiya_reply_t* hiya_6_svc( prod_class_t *offered, struct svc_req *rqstp) { const char* const pqfname = getQueuePath(); static hiya_reply_t reply; SVCXPRT * const xprt = rqstp->rq_xprt; struct sockaddr_in *upAddr = (struct sockaddr_in*) svc_getcaller(xprt); const char *upName = hostbyaddr(upAddr); int error; int isPrimary; unsigned int maxHereis; static prod_class_t *accept; /* * Open the product-queue for writing. It will be closed by cleanup() * during process termination. */ if (pq) { (void) pq_close(pq); pq = NULL; } error = pq_open(pqfname, PQ_DEFAULT, &pq); if (error) { err_log_and_free(ERR_NEW2(error, NULL, "Couldn't open product-queue \"%s\" for writing: %s", pqfname, PQ_CORRUPT == error ? "The product-queue is inconsistent" : strerror(error)), ERR_FAILURE); svcerr_systemerr(xprt); svc_destroy(xprt); exit(error); } /* else */ error = down6_init(upName, upAddr, pqfname, pq); if (error) { log_error_q("Couldn't initialize downstream LDM"); svcerr_systemerr(xprt); svc_destroy(xprt); exit(error); } else { log_info_q("Downstream LDM initialized"); } /* * The previous "accept" is freed here -- rather than freeing the * soon-to-be-allocated "accept" at the end of its block -- because it can * be used in the reply. */ if (accept) { free_prod_class(accept); /* NULL safe */ accept = NULL; } error = lcf_reduceToAcceptable(upName, inet_ntoa(upAddr->sin_addr), offered, &accept, &isPrimary); maxHereis = isPrimary ? UINT_MAX : 0; if (error) { log_syserr_q("Couldn't validate HIYA"); svcerr_systemerr(xprt); svc_destroy(xprt); exit(error); } else { if (log_is_enabled_debug) log_debug("intersection: %s", s_prod_class(NULL, 0, accept)); if (accept->psa.psa_len == 0) { log_warning_q("Empty intersection of HIYA offer from %s (%s) and ACCEPT " "entries", upName, s_prod_class(NULL, 0, offered)); svcerr_weakauth(xprt); svc_destroy(xprt); exit(0); } else { error = down6_set_prod_class(accept); if (error) { if (DOWN6_SYSTEM_ERROR == error) { log_syserr_q("Couldn't set product class: %s", s_prod_class(NULL, 0, accept)); } else { log_error_q("Couldn't set product class: %s", s_prod_class(NULL, 0, accept)); } svcerr_systemerr(xprt); svc_destroy(xprt); exit(EXIT_FAILURE); } /* else */ if (clss_eq(offered, accept)) { log_notice_q("hiya6: %s", s_prod_class(NULL, 0, offered)); reply.code = OK; reply.hiya_reply_t_u.max_hereis = maxHereis; } else { if (log_is_enabled_info) { char off[512]; char acc[512]; (void) s_prod_class(off, sizeof(off), offered), (void) s_prod_class( acc, sizeof(acc), accept); log_info_q("hiya6: RECLASS: %s -> %s", off, acc); } reply.code = RECLASS; reply.hiya_reply_t_u.feedPar.prod_class = accept; reply.hiya_reply_t_u.feedPar.max_hereis = maxHereis; } } /* product-intersection != empty set */ } /* successful acl_check_hiya() */ return &reply; }
int generate_retrans_rqst(ACQ_TABLE *p_acqtable, long first_prod_seqno, long last_prod_seqno, int rqst_cause) { static char FNAME[] = "generate_retrans_rqst"; static ulong request_numb; int prod_lost; PIPE_RETRANSMIT_HDR pipe_request_hdr; PIPE_RETRANSMIT_HDR *p_pipe_requesthdr; size_t bytes_written; time_t tmp_time; request_numb++; prod_lost = (last_prod_seqno - first_prod_seqno) + 1; if(prod_lost < 0) prod_lost = 0; p_acqtable->proc_tot_prods_retrans_rqstd += prod_lost; time(&p_acqtable->proc_last_retrans_rqst); if(global_retransmitpipe_fd <= 0){ /* Unable to open/write to pipe */ log_error_q("Unable to open or write to pipe %s \n",DEFAULT_RETRANSMIT_PIPENAME); return (0); } p_pipe_requesthdr = &pipe_request_hdr; p_pipe_requesthdr->pipe_ctl_flag = 0; if(p_acqtable->proc_retransmit_ctl_flag & ENABLE_RETRANS_XMIT_RQST){ p_pipe_requesthdr->pipe_ctl_flag |= ENABLE_RETRANS_XMIT_RQST; } p_pipe_requesthdr->pipe_request_numb = request_numb; p_pipe_requesthdr->pipe_link_id = p_acqtable->link_id; p_pipe_requesthdr->pipe_channel_type = p_acqtable->proc_base_channel_type_last; p_pipe_requesthdr->pipe_first_prod_seqno = (int)first_prod_seqno; p_pipe_requesthdr->pipe_last_prod_seqno = (int)last_prod_seqno; p_pipe_requesthdr->pipe_run_numb = (int) p_acqtable->proc_prod_run_id; p_pipe_requesthdr->pipe_cpio_addr = (int) global_cpio_addr; time(&tmp_time); p_pipe_requesthdr->pipe_request_time = (int) tmp_time; p_pipe_requesthdr->pipe_delay_send = p_acqtable->proc_retransmit_delay_send; p_pipe_requesthdr->pipe_request_cause = RETRANS_RQST_CAUSE_RCV_ERR; log_debug("pipe_request_numb = %ld | ctl_flag = %d | link_id = %d | channel_type = %d | rqst cause = %d \n", p_pipe_requesthdr->pipe_request_numb,p_pipe_requesthdr->pipe_ctl_flag,p_pipe_requesthdr->pipe_link_id, p_pipe_requesthdr->pipe_channel_type,p_pipe_requesthdr->pipe_request_cause); log_debug("cpio addr = %ld | rqst time = %ld | first prod seqno = [%d-%ld] | last prod seqno = %ld | run number = %d | delay_send = %d \n", p_pipe_requesthdr->pipe_cpio_addr, p_pipe_requesthdr->pipe_request_time,p_pipe_requesthdr->pipe_first_prod_seqno,first_prod_seqno, p_pipe_requesthdr->pipe_last_prod_seqno,p_pipe_requesthdr->pipe_run_numb,p_pipe_requesthdr->pipe_delay_send); /* Now write to pipe */ if((bytes_written = write(global_retransmitpipe_fd, p_pipe_requesthdr, sizeof(PIPE_RETRANSMIT_HDR))) != sizeof(PIPE_RETRANSMIT_HDR)) { log_error_q("FAIL write#%ld pipe[%d] tot(%ld) %s link[%d] prod(%ld-%ld)\n", p_pipe_requesthdr->pipe_request_numb, global_retransmitpipe_fd, p_acqtable->proc_tot_prods_retrans_rqstd, GET_SBN_TYP_NAME(p_pipe_requesthdr->pipe_channel_type), p_pipe_requesthdr->pipe_link_id, first_prod_seqno, last_prod_seqno); } else { if(first_prod_seqno != last_prod_seqno) { log_info_q("OK rqst#%ld tot(%ld) %s link[%d] prod(%ld-%ld)\n", request_numb, p_acqtable->proc_tot_prods_retrans_rqstd, GET_SBN_TYP_NAME(p_pipe_requesthdr->pipe_channel_type), p_pipe_requesthdr->pipe_link_id, first_prod_seqno, last_prod_seqno); } else { log_info_q("OK rqst#%ld tot(%ld) %s link[%d] prod(%ld)\n", request_numb, p_acqtable->proc_tot_prods_retrans_rqstd, GET_SBN_TYP_NAME(p_pipe_requesthdr->pipe_channel_type), p_pipe_requesthdr->pipe_link_id, last_prod_seqno); } } return(0); }
/* * Sends a list of files to the LDM as data-products. * * Arguments: * ldmProxy The LDM proxy data-structure. * offer The description of the class of data-products that this * process is willing to send. * origin The identifier of the host that created the * data-products (typically the host running this program). * seq_start The starting value of the data-product sequence number. * nfiles The number of files to send. * filenames The pathnames of the files to send. * * Returns: * 0 Success. * SYSTEM_ERROR O/S failure. "log_add()" called. * CONNECTION_ABORTED The connection was aborted. "log_add()" * called. */ static int ldmsend( LdmProxy* ldmProxy, prod_class_t* offer, char* origin, int seq_start, int nfiles, char* filenames[]) { int status = 0; char* filename; int fd; struct stat statb; prod_info info; MD5_CTX* md5ctxp = NULL; prod_class_t* want; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if (md5ctxp == NULL) { log_syserr_q("new_md5_CTX failed"); return SYSTEM_ERROR; } status = lp_hiya(ldmProxy, offer, &want); if (status != 0) { status = CONNECTION_ABORTED; } else { /* These members are constant over the loop. */ info.origin = origin; info.feedtype = offer->psa.psa_val->feedtype; for (info.seqno = seq_start; exitIfDone(1) && nfiles > 0; filenames++, nfiles--, info.seqno++) { filename = *filenames; info.ident = filename; /* * ?? This could be the creation time of the file. */ (void) set_timestamp(&info.arrival); /* * Checks 'arrival', 'feedtype', and 'ident' * against what the other guy has said he wants. */ if (!prodInClass(offer, &info)) { log_info_q("Not going to send %s", filename); continue; } if (!prodInClass(want, &info)) { log_info_q("%s doesn't want %s", lp_host(ldmProxy), filename); continue; } fd = open(filename, O_RDONLY, 0); if (fd == -1) { log_syserr_q("open: %s", filename); continue; } if (fstat(fd, &statb) == -1) { log_syserr_q("fstat: %s", filename); (void) close(fd); continue; } log_info_q("Sending %s, %d bytes", filename, statb.st_size); /* These members, and seqno, vary over the loop. */ if (fd_md5(md5ctxp, fd, statb.st_size, info.signature) != 0) { (void) close(fd); continue; } if (lseek(fd, 0, SEEK_SET) == (off_t)-1) { log_syserr_q("rewind: %s", filename); (void) close(fd); continue; } info.sz = (u_int)statb.st_size; (void)exitIfDone(1); status = send_product(ldmProxy, fd, &info); (void) close(fd); if (0 != status) { log_add("Couldn't send file \"%s\" to LDM", filename); break; } } /* file loop */ if (lp_flush(ldmProxy)) log_add("Couldn't flush connection"); free_prod_class(want); } /* HIYA succeeded */ free_MD5_CTX(md5ctxp); return status; }
int main( int ac, char *av[] ) { const char* const progname = basename(av[0]); int useProductID = FALSE; int signatureFromId = FALSE; char *productID = NULL; int multipleFiles = FALSE; char identifier[KEYSIZE]; int status; int seq_start = 0; enum ExitCode { exit_success = 0, /* all files inserted successfully */ exit_system = 1, /* operating-system failure */ exit_pq_open = 2, /* couldn't open product-queue */ exit_infile = 3, /* couldn't process input file */ exit_dup = 4, /* input-file already in product-queue */ exit_md5 = 6 /* couldn't initialize MD5 processing */ } exitCode = exit_success; (void)log_init(progname); #if !USE_MMAP pqeIndex = PQE_NONE; #endif { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 0; /* Suppress getopt(3) error messages */ while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF) switch (ch) { case 'i': signatureFromId = 1; break; case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': (void)log_set_destination(optarg); break; case 'q': setQueuePath(optarg); break; case 's': seq_start = atoi(optarg); break; case 'f': feedtype = atofeedtypet(optarg); if(feedtype == NONE) { fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg); usage(progname); } break; case 'p': useProductID = TRUE; productID = optarg; break; case ':': { log_add("Option \"-%c\" requires an operand", optopt); usage(progname); } /* no break */ default: log_add("Unknown option: \"%c\"", optopt); usage(progname); /* no break */ } ac -= optind; av += optind ; if(ac < 1) usage(progname); } const char* const pqfname = getQueuePath(); /* * register exit handler */ if(atexit(cleanup) != 0) { log_syserr_q("atexit"); exit(exit_system); } /* * set up signal handlers */ set_sigactions(); /* * who am i, anyway */ (void) strncpy(myname, ghostname(), sizeof(myname)); myname[sizeof(myname)-1] = 0; /* * open the product queue */ if(status = pq_open(pqfname, PQ_DEFAULT, &pq)) { if (PQ_CORRUPT == status) { log_error_q("The product-queue \"%s\" is inconsistent\n", pqfname); } else { log_error_q("pq_open: \"%s\" failed: %s", pqfname, status > 0 ? strerror(status) : "Internal error"); } exit(exit_pq_open); } { char *filename; int fd; struct stat statb; product prod; MD5_CTX *md5ctxp = NULL; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX(); if(md5ctxp == NULL) { log_syserr_q("new_md5_CTX failed"); exit(exit_md5); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; if (ac > 1) { multipleFiles = TRUE; } for(prod.info.seqno = seq_start ; ac > 0 ; av++, ac--, prod.info.seqno++) { filename = *av; fd = open(filename, O_RDONLY, 0); if(fd == -1) { log_syserr_q("open: %s", filename); exitCode = exit_infile; continue; } if( fstat(fd, &statb) == -1) { log_syserr_q("fstat: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } /* Determine what to use for product identifier */ if (useProductID) { if (multipleFiles) { sprintf(identifier,"%s.%d", productID, prod.info.seqno); prod.info.ident = identifier; } else prod.info.ident = productID; } else prod.info.ident = filename; prod.info.sz = statb.st_size; prod.data = NULL; /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { log_syserr_q("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } #if USE_MMAP prod.data = mmap(0, prod.info.sz, PROT_READ, MAP_PRIVATE, fd, 0); if(prod.data == MAP_FAILED) { log_syserr_q("mmap: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); (void)exitIfDone(1); if (status != 0) { log_syserr_q("mm_md5: %s", filename); (void) munmap(prod.data, prod.info.sz); (void) close(fd); exitCode = exit_infile; continue; } /* These members, and seqno, vary over the loop. */ status = set_timestamp(&prod.info.arrival); if(status != ENOERR) { log_syserr_q("set_timestamp: %s, filename"); exitCode = exit_infile; continue; } /* * Do the deed */ status = pq_insert(pq, &prod); switch (status) { case ENOERR: /* no error */ if(log_is_enabled_info) log_info_q("%s", s_prod_info(NULL, 0, &prod.info, log_is_enabled_debug)) ; break; case PQUEUE_DUP: log_error_q("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case PQUEUE_BIG: log_error_q("Product too big for queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_infile; break; case ENOMEM: log_error_q("queue full?"); exitCode = exit_system; break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: log_error_q("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); break; } (void) munmap(prod.data, prod.info.sz); #else // USE_MMAP above; !USE_MMAP below status = signatureFromId ? mm_md5(md5ctxp, prod.info.ident, strlen(prod.info.ident), prod.info.signature) : fd_md5(md5ctxp, fd, statb.st_size, prod.info.signature); (void)exitIfDone(1); if (status != 0) { log_syserr_q("xx_md5: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } if(lseek(fd, 0, SEEK_SET) == (off_t)-1) { log_syserr_q("rewind: %s", filename); (void) close(fd); exitCode = exit_infile; continue; } pqeIndex = PQE_NONE; status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex); if(status != ENOERR) { log_syserr_q("pqe_new: %s", filename); exitCode = exit_infile; } else { ssize_t nread = read(fd, prod.data, prod.info.sz); (void)exitIfDone(1); if (nread != prod.info.sz) { log_syserr_q("read %s %u", filename, prod.info.sz); status = EIO; } else { status = pqe_insert(pq, pqeIndex); pqeIndex = PQE_NONE; switch (status) { case ENOERR: /* no error */ if(ulogIsVerbose()) log_info_q("%s", s_prod_info(NULL, 0, &prod.info, log_is_enabled_debug)) ; break; case PQUEUE_DUP: log_error_q("Product already in queue: %s", s_prod_info(NULL, 0, &prod.info, 1)); exitCode = exit_dup; break; case ENOMEM: log_error_q("queue full?"); break; case EINTR: #if defined(EDEADLOCK) && EDEADLOCK != EDEADLK case EDEADLOCK: /*FALLTHROUGH*/ #endif case EDEADLK: /* TODO: retry ? */ /*FALLTHROUGH*/ default: log_error_q("pq_insert: %s", status > 0 ? strerror(status) : "Internal error"); } } /* data read into `pqeIndex` region */ if (status != ENOERR) { (void)pqe_discard(pq, pqeIndex); pqeIndex = PQE_NONE; } } /* `pqeIndex` region allocated */ #endif (void) close(fd); } /* input-file loop */ free_MD5_CTX(md5ctxp); } /* code block */ exit(exitCode); }
/* * Reads one record's worth of data from the FIFO and writes it to a * client-supplied buffer. Blocks until data is available. * * Arguments: * shm Pointer to the shared-memory FIFO data-structure. The * FIFO must be unlocked. * data Pointer to the buffer into which to put data from the * FIFO. * sz The size of the buffer in bytes. * nbytes Pointer to the memory location to be set to the number * of bytes read. * Returns: * 0 Success. "*nbytes" is set to the number of bytes read. * ECANCELED Operating-system failure. Error-message logged. * EINVAL "shm" uninitialized. Error-message logged. * EINVAL "sz" is non-positive. Error-message logged. * EINVAL The buffer is too small for the record's data. No data * is read. Error-message logged. * EIO FIFO is corrupt. Error-message logged. * Raises: * SIGSEGV if "shm" is NULL * SIGSEGV if "data" is NULL * SIGSEGV if "nbytes" is NULL * SIGABRT if "shm" is uninitialized */ int shmfifo_get( const struct shmhandle* const shm, void* const data, const int sz, int* const nbytes) { int status; if (sz <= 0) { log_error_q("Non-positive number of bytes to read: %d", sz); status = EINVAL; } else { int loggedEmptyFifo = 0; if ((status = shmfifo_lock(shm)) == 0) { shmfifo_printmemstatus(shm); for (status = 0; shmfifo_ll_memused(shm) == 0; ) { if (!loggedEmptyFifo) { log_info_q("shmfifo_get(): FIFO is empty"); loggedEmptyFifo = 1; } if ((status = shmfifo_wait_reader(shm)) != 0) { break; } } if (0 == status) { struct shmbh header; if (shmfifo_ll_memused(shm) < (int)sizeof(header)) { log_error_q("Insufficient data for a record: " "should be at least %d bytes; was %d bytes", sizeof(header), shmfifo_ll_memused(shm)); shmfifo_print(shm); status = EINVAL; } else { shmfifo_ll_get(shm, &header, sizeof(header)); if (header.canary != 0xDEADBEEF) { log_error_q("Invalid header sentinel: 0x%X", header.canary); status = EIO; } else if (shmfifo_ll_memused(shm) < header.sz) { log_error_q("Inconsistent data-length of record: " "expected %d bytes; encountered %d bytes", header.sz, shmfifo_ll_memused(shm)); shmfifo_print(shm); status = EIO; } else if (header.sz > sz) { log_error_q("Client-supplied buffer too small: " "need %d bytes; %d bytes supplied", header.sz, sz); shmfifo_ll_hrewind(shm); status = EINVAL; } else { shmfifo_ll_get(shm, data, header.sz); if (loggedEmptyFifo) { log_info_q("shmfifo_get(): " "Got %d bytes of data from FIFO", header.sz); } shmfifo_printmemstatus(shm); if ((status = shmfifo_notify_writer(shm)) == 0) { *nbytes = header.sz; } } } } /* FIFO has data */ int tmpStatus = shmfifo_unlock(shm); if (status == 0) status = tmpStatus; } /* shared-memory FIFO locked */ } return status; }
/* * Writes data to the shared-memory FIFO. * * Arguments: * shm Pointer to the shared-memory FIFO data-structure. * data Pointer to the data to be written. * sz The amount of data to be written in bytes. * Returns: * 0 Success. * E2BIG "sz" is larger than the FIFO can handle. Error-message * logged. * ECANCELED Operating-system failure. Error-message logged. * EIO I/O error. Error-message logged. * EINVAL "sz" is negative. Error-message logged. * EINVAL "shm" uninitialized. Error-message logged. */ int shmfifo_put( const struct shmhandle* const shm, void* const data, const int sz) { int status; if (0 > sz) { log_error_q("Invalid size argument: %d", sz); status = EINVAL; } else { if ((status = shmfifo_lock(shm)) == 0) { struct shmbh header; const size_t totalBytesToWrite = sz + sizeof(header); size_t maxSize; shmfifo_printmemstatus(shm); maxSize = shmfifo_ll_memused(shm) + shmfifo_ll_memfree(shm); if (maxSize < totalBytesToWrite) { log_error_q("Record bigger than entire FIFO: " "record is %lu bytes; FIFO capacity is %lu bytes", totalBytesToWrite, maxSize); status = E2BIG; } else { int loggedNoRoom = 0; int freeSpace; status = 0; /* * Wait for the FIFO to have room for the data. */ while ((freeSpace = shmfifo_ll_memfree(shm)) <= totalBytesToWrite) { if (!loggedNoRoom) { log_error_q("No room in FIFO: " "need %d bytes; only %d bytes available. " "Waiting...", totalBytesToWrite, freeSpace); loggedNoRoom = 1; } if ((status = shmfifo_wait_writer(shm)) != 0) { break; } } if (0 == status) { header.sz = sz; header.canary = 0xDEADBEEF; shmfifo_ll_put(shm, &header, sizeof(header)); shmfifo_ll_put(shm, data, sz); if (loggedNoRoom) { log_info_q("shmfifo_put(): Wrote %d bytes to FIFO", totalBytesToWrite); } status = shmfifo_notify_reader(shm); } } int tmpStatus = shmfifo_unlock(shm); if (status == 0) status = tmpStatus; } /* shared-memory FIFO locked */ } // `sz` is valid return status; }
int main (int ac, char *av[]) { char *progname = av[0]; int status; int seq_start = 0; stat_info *sinfo, *shead = NULL, *slast = NULL; int statusoff=0; /* * Set up error logging */ if (log_init(progname)) { log_syserr("Couldn't initialize logging module"); exit(1); } const char* pqfname = getQueuePath(); /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv ("LDMPQFNAME"); if (ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 1; while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': if (log_set_destination(optarg)) { log_syserr("Couldn't set logging destination to \"%s\"", optarg); exit(1); } break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi (optarg); break; case 'f': feedtype = atofeedtypet (optarg); if (feedtype == NONE) { fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg); usage (progname); } break; case 'S': statusoff=1; break; case '?': usage (progname); break; } setQueuePath(pqfname); ac -= optind; av += optind; if (ac < 1) usage (progname); } /* * register exit handler */ if (atexit (cleanup) != 0) { log_syserr ("atexit"); exit (1); } /* * set up signal handlers */ set_sigactions (); /* * who am i, anyway */ (void) strcpy (myname, ghostname ()); /* * open the product queue */ if ((status = pq_open (pqfname, PQ_DEFAULT, &pq))) { if (status > 0) { log_add_syserr("\"%s\" failed", pqfname); log_flush_error(); } else { log_error_q("\"%s\" failed: %s", pqfname, "Internal error"); } exit (2); } { char *filename; int fd; struct stat statb; product prod; unsigned char *prodmmap; MD5_CTX *md5ctxp = NULL; int gversion; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX (); if (md5ctxp == NULL) { log_syserr ("new_md5_CTX failed"); exit (6); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; prod.info.seqno = seq_start; /* * Open the file to be inserted and process */ while (ac > 0) { long insert_sum = 0; long sinfo_cnt = 0; long stat_size = 0; filename = *av; av++; ac--; log_notice_q ("open and memorymap %s\0", filename); fd = open (filename, O_RDONLY, 0); if (fd == -1) { log_syserr ("open: %s", filename); continue; } if (fstat (fd, &statb) == -1) { log_syserr ("fstat: %s", filename); (void) close (fd); continue; } if ((prodmmap = (unsigned char *) mmap (0, statb.st_size, PROT_READ, MAP_PRIVATE, fd, 0)) == MAP_FAILED) { log_syserr ("allocation failed"); } else { int GRIBDONE = 0; off_t griboff = 0; size_t griblen = 0; log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size); while (!GRIBDONE) { log_debug("griboff %d\0", (int) griboff); /* get offset of next grib product */ status = get_grib_info (prodmmap, statb.st_size, &griboff, &griblen, &gversion); switch (status) { case 0: prod.data = prodmmap + griboff; prod.info.sz = griblen; /* * revised MD5 calculation...using filename * to allow duplicate products in different files. */ MD5Init (md5ctxp); MD5Update (md5ctxp, (void *)filename, strlen(filename)); /*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/ if ( prod.info.sz > 10000 ) MD5Update (md5ctxp, (void *)prod.data, 10000); else MD5Update (md5ctxp, (void *)prod.data, prod.info.sz); MD5Final (prod.info.signature, md5ctxp); /*if (mm_md5 (md5ctxp, prod.data, prod.info.sz, prod.info.signature) != 0) { log_error_q ("could not compute MD5\0"); } else { */ prod.info.ident = (char *) malloc (KEYSIZE + 1); get_gribname (gversion, prod.data, prod.info.sz, filename, prod.info.seqno, prod.info.ident); /* * Do the deed */ status = set_timestamp (&prod.info.arrival); if (status != ENOERR) { log_add_syserr("could not set timestamp"); log_flush_error(); } /* * Insert the product */ status = pq_insert (pq, &prod); log_info_q ("%d %s\0", status, prod.info.ident); if ( status == ENOERR ) insert_sum += prod.info.sz; if (! statusoff ) { /* * Log this status */ sinfo_cnt++; sinfo = (stat_info *)malloc(sizeof(stat_info)); sinfo->insertstatus = status; sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1); strcpy(sinfo->prodname, prod.info.ident); sinfo->seqno = prod.info.seqno; sinfo->prodsz = prod.info.sz; sinfo->next = NULL; stat_size += strlen(sinfo->prodname); if(shead == NULL) { shead = sinfo; slast = sinfo; } else { slast->next = sinfo; slast = sinfo; } } /*}*/ griboff += griblen; prod.info.seqno++; break; case -1: GRIBDONE = 1; break; case -2: log_error_q ("truncated grib file at: %d", prod.info.seqno); GRIBDONE = 1; break; case -7: log_error_q ("End sequence 7777 not found where expected: %d", prod.info.seqno); griboff += griblen; log_error_q("resume looking at %d\0",griboff); break; default: log_error_q ("unknown error %d\0", status); griboff += griblen; if (griboff >= statb.st_size) GRIBDONE = 1; break; } if (griboff >= statb.st_size) GRIBDONE = 1; } log_notice_q ("munmap\0"); (void) munmap ((void *)prodmmap, statb.st_size); if ( stat_size != 0 ) /* * Add a status message to product queue */ { char *statusmess; log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt); statusmess = calloc((30 * sinfo_cnt) + stat_size + strlen(filename) + 128, sizeof(char)); if(statusmess == NULL) { log_syserr("could not malloc status message %ld\0", stat_size); } else { char tmpprod[512]; sinfo = shead; slast = NULL; status = set_timestamp(&prod.info.arrival); /* ctime ends with \n\0" */ sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n", filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec), insert_sum,(long)statb.st_size); while(sinfo != NULL) { sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus, sinfo->seqno,sinfo->prodsz,sinfo->prodname); strcat(statusmess,tmpprod); slast = sinfo; sinfo = sinfo->next; free(slast->prodname); free(slast); } shead = NULL; sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno); prod.info.ident = tmpprod; prod.data = statusmess; prod.info.sz = strlen(statusmess); status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); status = set_timestamp(&prod.info.arrival); status = pq_insert(pq, &prod); if(log_is_enabled_info) log_info_q("%s", s_prod_info(NULL, 0, &prod.info, log_is_enabled_debug)) ; free(statusmess); prod.info.seqno++; } } } (void) close (fd); } } exit(0); }