int do_prod_lost( BUFF_HDR *buff_hdr, ACQ_TABLE *acq_tbl)
{
long prod_errors;
if(acq_tbl->proc_base_prod_seqno_last == 0){
prod_errors = 0;
}else{
prod_errors = (buff_hdr->proc_prod_seqno - (acq_tbl->proc_base_prod_seqno_last + 1));
}
if(prod_errors <= 0){
prod_errors = 0;
}else{
/* Now generate retransmission request */
generate_retrans_rqst(acq_tbl,(acq_tbl->proc_base_prod_seqno_last + 1),
(buff_hdr->proc_prod_seqno-1),
RETRANS_RQST_CAUSE_RCV_ERR);
}
acq_tbl->proc_tot_prods_lost_errs += prod_errors;
/* Need to log prod_errors */
if(acq_tbl->proc_base_prod_seqno_last == 0){
log_info_q("LOST=%ld total(%ld) %s prod(%ld) prod_seqno was RESET to 0 \n",
prod_errors,acq_tbl->proc_tot_prods_lost_errs,
GET_PROD_TYPE_NAME(buff_hdr->proc_prod_type),buff_hdr->proc_prod_seqno);
}else{
log_info_q("LOST=%ld total(%ld) %s prod(%ld) expect prod(%ld)\n",
prod_errors, acq_tbl->proc_tot_prods_lost_errs,
GET_PROD_TYPE_NAME(buff_hdr->proc_prod_type),buff_hdr->proc_prod_seqno,
acq_tbl->proc_base_prod_seqno_last + 1);
}
return (0);
}
/*
* The RPC dispatch routine for this program.
* Registered as a callback by svc_register() below.
* Note that only NULLPROC and NOTIFICATION rpc procs are
* handled by this program.
*/
static void
notifymeprog_5(struct svc_req *rqstp, SVCXPRT *transp)
{
prod_info notice;
switch (rqstp->rq_proc) {
case NULLPROC:
(void)svc_sendreply(transp, (xdrproc_t)xdr_void, (caddr_t)NULL);
return;
case NOTIFICATION:
(void) memset((char*)¬ice, 0, sizeof(notice));
if (!svc_getargs(transp, (xdrproc_t)xdr_prod_info,
(caddr_t)¬ice))
{
svcerr_decode(transp);
return;
}
(void)exitIfDone(0);
/*
* Update the request filter with the timestamp
* we just recieved.
* N.B.: There can still be duplicates after
* a reconnect.
*/
clss.from = notice.arrival;
timestamp_incr(&clss.from);
/*
* your code here, example just logs it
*/
log_info_q("%s", s_prod_info(NULL, 0, ¬ice,
log_is_enabled_debug));
if(!svc_sendreply(transp, (xdrproc_t)xdr_ldm_replyt,
(caddr_t) &reply))
{
svcerr_systemerr(transp);
}
(void)exitIfDone(0);
if(!svc_freeargs(transp, xdr_prod_info, (caddr_t) ¬ice)) {
log_error_q("unable to free arguments");
exit(1);
}
/* no break */
default:
svcerr_noproc(transp);
return;
}
}
/**
* There is data available on the feed. Read it into the buffer
* then deal with what we got.
*
* @param ifd [in] File-descriptor of the input data-feed
* @retval 0 Success
* @retval ENOMEM Out of memory
* @retval ENODATA End of input data-feed
*/
int
feedTheXbuf(const int ifd)
{
int status;
size_t nn = 0;
/* space available in buffer */
ptrdiff_t remaining = (ptrdiff_t)theBuf->bufsiz - (theBuf->get - theBuf->base);
if (remaining <= CHUNKSIZE) {
if (theBuf->bufsiz >= maxProductSize) {
log_warning_q(
"Data-product would exceed %lu bytes. Resetting input buffer.",
maxProductSize);
justify_xbuf(theBuf, 0);
}
log_info_q("Expanding input buffer size to %lu\n",
(unsigned long)(2 * theBuf->bufsiz));
theBuf = expand_xbuf(theBuf, theBuf->bufsiz);
if (theBuf == NULL) {
status = errno == 0 ? ENOMEM : errno;
log_add_syserr("expand_xbuf");
log_flush_error();
return status;
}
}
status = (*read_feed)(ifd, (char *)theBuf->put, CHUNKSIZE, &nn);
if(status != ENOERR)
{
log_add_errno(status, "read_feed");
log_flush_error();
return status;
}
/* else */
if(nn == 0)
return ENODATA; /* end of file */
/* else */
/* usual case */
/* assert(nn > 0); */
theBuf->cnt += nn;
theBuf->put += nn;
return ENOERR;
}
int main(int ac, char *av[])
{
int pflags = PQ_NOCLOBBER;
off_t initialsz = 0;
size_t nproducts = 0;
pqueue *pq = NULL;
int errnum = 0;
/*
* initialize logger
*/
(void)log_init(av[0]);
int ch;
char *qopt = NULL;
char *sopt = NULL;
char *Sopt = NULL;
extern char *optarg;
extern int optind;
const char* pqfname = getQueuePath();
while ((ch = getopt(ac, av, "xvcfq:s:S:l:")) != EOF)
switch (ch) {
case 'v':
if (!log_is_enabled_info)
(void)log_set_level(LOG_LEVEL_INFO);
break;
case 'c':
pflags &= ~PQ_NOCLOBBER;
break;
case 'f':
pflags |= PQ_SPARSE;
break;
case 's':
sopt = optarg;
break;
case 'S':
Sopt = optarg;
break;
case 'q':
qopt = optarg;
break;
case 'x':
(void)log_set_level(LOG_LEVEL_DEBUG);
break;
case 'l':
(void)log_set_destination(optarg);
break;
case '?':
usage(av[0]);
break;
}
if(ac - optind > 1)
{
if(sopt)
usage(av[0]);
sopt = av[ac - 2];
}
if(ac - optind > 0)
{
if(qopt)
usage(av[0]);
qopt = av[ac - 1];
}
if(qopt) {
pqfname = qopt ;
setQueuePath(qopt);
}
if (sopt) {
char* cp;
int exponent = 0;
errno = 0;
initialsz = strtol(sopt, &cp, 0);
if (0 != errno) {
initialsz = 0; /* trigger error below */
}
else {
switch (*cp) {
case 0:
break;
case 'k':
case 'K':
exponent = 1;
break;
case 'm':
case 'M':
exponent = 2;
break;
case 'g':
case 'G':
exponent = 3;
break;
default:
initialsz = 0; /* trigger error below */
break;
}
if (0 < initialsz) {
int i;
for (i = 0; i < exponent; i++) {
initialsz *= 1000;
if (0 >= initialsz) {
fprintf(stderr, "Size \"%s\" too big\n", sopt);
usage(av[0]);
}
}
}
}
}
if(initialsz <= 0)
{
if(sopt)
fprintf(stderr, "Illegal size \"%s\"\n", sopt);
else
fprintf(stderr, "No size specified\n");
usage(av[0]);
}
if(Sopt != NULL)
{
nproducts = (size_t)atol(Sopt);
if(nproducts == 0)
{
fprintf(stderr, "Illegal nproducts \"%s\"\n", Sopt);
}
}
else
{
#define PQ_AVG_PRODUCT_SIZE 51000 // approximate mean size on 2014-08-21
/* For default number of product slots, use average product size estimate */
nproducts = initialsz/PQ_AVG_PRODUCT_SIZE;
}
log_info_q("Creating %s, %ld bytes, %ld products.\n",
pqfname, (long)initialsz, (long)nproducts);
errnum = pq_create(pqfname, 0666, pflags,
0, initialsz, nproducts, &pq);
if(errnum)
{
fprintf(stderr, "%s: create \"%s\" failed: %s\n",
av[0], pqfname, strerror(errnum));
exit(1);
}
(void)pq_close(pq);
return(0);
}
hiya_reply_t*
hiya_6_svc(
prod_class_t *offered,
struct svc_req *rqstp)
{
const char* const pqfname = getQueuePath();
static hiya_reply_t reply;
SVCXPRT * const xprt = rqstp->rq_xprt;
struct sockaddr_in *upAddr = (struct sockaddr_in*) svc_getcaller(xprt);
const char *upName = hostbyaddr(upAddr);
int error;
int isPrimary;
unsigned int maxHereis;
static prod_class_t *accept;
/*
* Open the product-queue for writing. It will be closed by cleanup()
* during process termination.
*/
if (pq) {
(void) pq_close(pq);
pq = NULL;
}
error = pq_open(pqfname, PQ_DEFAULT, &pq);
if (error) {
err_log_and_free(ERR_NEW2(error, NULL,
"Couldn't open product-queue \"%s\" for writing: %s",
pqfname,
PQ_CORRUPT == error
? "The product-queue is inconsistent"
: strerror(error)), ERR_FAILURE);
svcerr_systemerr(xprt);
svc_destroy(xprt);
exit(error);
}
/* else */
error = down6_init(upName, upAddr, pqfname, pq);
if (error) {
log_error_q("Couldn't initialize downstream LDM");
svcerr_systemerr(xprt);
svc_destroy(xprt);
exit(error);
}
else {
log_info_q("Downstream LDM initialized");
}
/*
* The previous "accept" is freed here -- rather than freeing the
* soon-to-be-allocated "accept" at the end of its block -- because it can
* be used in the reply.
*/
if (accept) {
free_prod_class(accept); /* NULL safe */
accept = NULL;
}
error = lcf_reduceToAcceptable(upName, inet_ntoa(upAddr->sin_addr), offered,
&accept, &isPrimary);
maxHereis = isPrimary ? UINT_MAX : 0;
if (error) {
log_syserr_q("Couldn't validate HIYA");
svcerr_systemerr(xprt);
svc_destroy(xprt);
exit(error);
}
else {
if (log_is_enabled_debug)
log_debug("intersection: %s", s_prod_class(NULL, 0, accept));
if (accept->psa.psa_len == 0) {
log_warning_q("Empty intersection of HIYA offer from %s (%s) and ACCEPT "
"entries", upName, s_prod_class(NULL, 0, offered));
svcerr_weakauth(xprt);
svc_destroy(xprt);
exit(0);
}
else {
error = down6_set_prod_class(accept);
if (error) {
if (DOWN6_SYSTEM_ERROR == error) {
log_syserr_q("Couldn't set product class: %s",
s_prod_class(NULL, 0, accept));
}
else {
log_error_q("Couldn't set product class: %s",
s_prod_class(NULL, 0, accept));
}
svcerr_systemerr(xprt);
svc_destroy(xprt);
exit(EXIT_FAILURE);
}
/* else */
if (clss_eq(offered, accept)) {
log_notice_q("hiya6: %s", s_prod_class(NULL, 0, offered));
reply.code = OK;
reply.hiya_reply_t_u.max_hereis = maxHereis;
}
else {
if (log_is_enabled_info) {
char off[512];
char acc[512];
(void) s_prod_class(off, sizeof(off), offered), (void) s_prod_class(
acc, sizeof(acc), accept);
log_info_q("hiya6: RECLASS: %s -> %s", off, acc);
}
reply.code = RECLASS;
reply.hiya_reply_t_u.feedPar.prod_class = accept;
reply.hiya_reply_t_u.feedPar.max_hereis = maxHereis;
}
} /* product-intersection != empty set */
} /* successful acl_check_hiya() */
return &reply;
}
int generate_retrans_rqst(ACQ_TABLE *p_acqtable,
long first_prod_seqno,
long last_prod_seqno,
int rqst_cause)
{
static char FNAME[] = "generate_retrans_rqst";
static ulong request_numb;
int prod_lost;
PIPE_RETRANSMIT_HDR pipe_request_hdr;
PIPE_RETRANSMIT_HDR *p_pipe_requesthdr;
size_t bytes_written;
time_t tmp_time;
request_numb++;
prod_lost = (last_prod_seqno - first_prod_seqno) + 1;
if(prod_lost < 0)
prod_lost = 0;
p_acqtable->proc_tot_prods_retrans_rqstd += prod_lost;
time(&p_acqtable->proc_last_retrans_rqst);
if(global_retransmitpipe_fd <= 0){
/* Unable to open/write to pipe */
log_error_q("Unable to open or write to pipe %s \n",DEFAULT_RETRANSMIT_PIPENAME);
return (0);
}
p_pipe_requesthdr = &pipe_request_hdr;
p_pipe_requesthdr->pipe_ctl_flag = 0;
if(p_acqtable->proc_retransmit_ctl_flag & ENABLE_RETRANS_XMIT_RQST){
p_pipe_requesthdr->pipe_ctl_flag |= ENABLE_RETRANS_XMIT_RQST;
}
p_pipe_requesthdr->pipe_request_numb = request_numb;
p_pipe_requesthdr->pipe_link_id = p_acqtable->link_id;
p_pipe_requesthdr->pipe_channel_type = p_acqtable->proc_base_channel_type_last;
p_pipe_requesthdr->pipe_first_prod_seqno = (int)first_prod_seqno;
p_pipe_requesthdr->pipe_last_prod_seqno = (int)last_prod_seqno;
p_pipe_requesthdr->pipe_run_numb = (int) p_acqtable->proc_prod_run_id;
p_pipe_requesthdr->pipe_cpio_addr = (int) global_cpio_addr;
time(&tmp_time);
p_pipe_requesthdr->pipe_request_time = (int) tmp_time;
p_pipe_requesthdr->pipe_delay_send = p_acqtable->proc_retransmit_delay_send;
p_pipe_requesthdr->pipe_request_cause = RETRANS_RQST_CAUSE_RCV_ERR;
log_debug("pipe_request_numb = %ld | ctl_flag = %d | link_id = %d | channel_type = %d | rqst cause = %d \n",
p_pipe_requesthdr->pipe_request_numb,p_pipe_requesthdr->pipe_ctl_flag,p_pipe_requesthdr->pipe_link_id,
p_pipe_requesthdr->pipe_channel_type,p_pipe_requesthdr->pipe_request_cause);
log_debug("cpio addr = %ld | rqst time = %ld | first prod seqno = [%d-%ld] | last prod seqno = %ld | run number = %d | delay_send = %d \n",
p_pipe_requesthdr->pipe_cpio_addr, p_pipe_requesthdr->pipe_request_time,p_pipe_requesthdr->pipe_first_prod_seqno,first_prod_seqno,
p_pipe_requesthdr->pipe_last_prod_seqno,p_pipe_requesthdr->pipe_run_numb,p_pipe_requesthdr->pipe_delay_send);
/* Now write to pipe */
if((bytes_written = write(global_retransmitpipe_fd,
p_pipe_requesthdr, sizeof(PIPE_RETRANSMIT_HDR))) !=
sizeof(PIPE_RETRANSMIT_HDR)) {
log_error_q("FAIL write#%ld pipe[%d] tot(%ld) %s link[%d] prod(%ld-%ld)\n",
p_pipe_requesthdr->pipe_request_numb,
global_retransmitpipe_fd,
p_acqtable->proc_tot_prods_retrans_rqstd,
GET_SBN_TYP_NAME(p_pipe_requesthdr->pipe_channel_type),
p_pipe_requesthdr->pipe_link_id,
first_prod_seqno, last_prod_seqno);
} else {
if(first_prod_seqno != last_prod_seqno) {
log_info_q("OK rqst#%ld tot(%ld) %s link[%d] prod(%ld-%ld)\n",
request_numb,
p_acqtable->proc_tot_prods_retrans_rqstd,
GET_SBN_TYP_NAME(p_pipe_requesthdr->pipe_channel_type),
p_pipe_requesthdr->pipe_link_id,
first_prod_seqno, last_prod_seqno);
} else {
log_info_q("OK rqst#%ld tot(%ld) %s link[%d] prod(%ld)\n",
request_numb,
p_acqtable->proc_tot_prods_retrans_rqstd,
GET_SBN_TYP_NAME(p_pipe_requesthdr->pipe_channel_type),
p_pipe_requesthdr->pipe_link_id,
last_prod_seqno);
}
}
return(0);
}
/*
* Sends a list of files to the LDM as data-products.
*
* Arguments:
* ldmProxy The LDM proxy data-structure.
* offer The description of the class of data-products that this
* process is willing to send.
* origin The identifier of the host that created the
* data-products (typically the host running this program).
* seq_start The starting value of the data-product sequence number.
* nfiles The number of files to send.
* filenames The pathnames of the files to send.
*
* Returns:
* 0 Success.
* SYSTEM_ERROR O/S failure. "log_add()" called.
* CONNECTION_ABORTED The connection was aborted. "log_add()" * called.
*/
static int
ldmsend(
LdmProxy* ldmProxy,
prod_class_t* offer,
char* origin,
int seq_start,
int nfiles,
char* filenames[])
{
int status = 0;
char* filename;
int fd;
struct stat statb;
prod_info info;
MD5_CTX* md5ctxp = NULL;
prod_class_t* want;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX();
if (md5ctxp == NULL)
{
log_syserr_q("new_md5_CTX failed");
return SYSTEM_ERROR;
}
status = lp_hiya(ldmProxy, offer, &want);
if (status != 0) {
status = CONNECTION_ABORTED;
}
else {
/* These members are constant over the loop. */
info.origin = origin;
info.feedtype = offer->psa.psa_val->feedtype;
for (info.seqno = seq_start; exitIfDone(1) && nfiles > 0;
filenames++, nfiles--, info.seqno++) {
filename = *filenames;
info.ident = filename;
/*
* ?? This could be the creation time of the file.
*/
(void) set_timestamp(&info.arrival);
/*
* Checks 'arrival', 'feedtype', and 'ident'
* against what the other guy has said he wants.
*/
if (!prodInClass(offer, &info)) {
log_info_q("Not going to send %s", filename);
continue;
}
if (!prodInClass(want, &info)) {
log_info_q("%s doesn't want %s", lp_host(ldmProxy), filename);
continue;
}
fd = open(filename, O_RDONLY, 0);
if (fd == -1) {
log_syserr_q("open: %s", filename);
continue;
}
if (fstat(fd, &statb) == -1) {
log_syserr_q("fstat: %s", filename);
(void) close(fd);
continue;
}
log_info_q("Sending %s, %d bytes", filename, statb.st_size);
/* These members, and seqno, vary over the loop. */
if (fd_md5(md5ctxp, fd, statb.st_size, info.signature) != 0) {
(void) close(fd);
continue;
}
if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
log_syserr_q("rewind: %s", filename);
(void) close(fd);
continue;
}
info.sz = (u_int)statb.st_size;
(void)exitIfDone(1);
status = send_product(ldmProxy, fd, &info);
(void) close(fd);
if (0 != status) {
log_add("Couldn't send file \"%s\" to LDM", filename);
break;
}
} /* file loop */
if (lp_flush(ldmProxy))
log_add("Couldn't flush connection");
free_prod_class(want);
} /* HIYA succeeded */
free_MD5_CTX(md5ctxp);
return status;
}
int main(
int ac,
char *av[]
)
{
const char* const progname = basename(av[0]);
int useProductID = FALSE;
int signatureFromId = FALSE;
char *productID = NULL;
int multipleFiles = FALSE;
char identifier[KEYSIZE];
int status;
int seq_start = 0;
enum ExitCode {
exit_success = 0, /* all files inserted successfully */
exit_system = 1, /* operating-system failure */
exit_pq_open = 2, /* couldn't open product-queue */
exit_infile = 3, /* couldn't process input file */
exit_dup = 4, /* input-file already in product-queue */
exit_md5 = 6 /* couldn't initialize MD5 processing */
} exitCode = exit_success;
(void)log_init(progname);
#if !USE_MMAP
pqeIndex = PQE_NONE;
#endif
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
opterr = 0; /* Suppress getopt(3) error messages */
while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF)
switch (ch) {
case 'i':
signatureFromId = 1;
break;
case 'v':
if (!log_is_enabled_info)
(void)log_set_level(LOG_LEVEL_INFO);
break;
case 'x':
(void)log_set_level(LOG_LEVEL_DEBUG);
break;
case 'l':
(void)log_set_destination(optarg);
break;
case 'q':
setQueuePath(optarg);
break;
case 's':
seq_start = atoi(optarg);
break;
case 'f':
feedtype = atofeedtypet(optarg);
if(feedtype == NONE)
{
fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg);
usage(progname);
}
break;
case 'p':
useProductID = TRUE;
productID = optarg;
break;
case ':': {
log_add("Option \"-%c\" requires an operand", optopt);
usage(progname);
}
/* no break */
default:
log_add("Unknown option: \"%c\"", optopt);
usage(progname);
/* no break */
}
ac -= optind; av += optind ;
if(ac < 1) usage(progname);
}
const char* const pqfname = getQueuePath();
/*
* register exit handler
*/
if(atexit(cleanup) != 0)
{
log_syserr_q("atexit");
exit(exit_system);
}
/*
* set up signal handlers
*/
set_sigactions();
/*
* who am i, anyway
*/
(void) strncpy(myname, ghostname(), sizeof(myname));
myname[sizeof(myname)-1] = 0;
/*
* open the product queue
*/
if(status = pq_open(pqfname, PQ_DEFAULT, &pq))
{
if (PQ_CORRUPT == status) {
log_error_q("The product-queue \"%s\" is inconsistent\n",
pqfname);
}
else {
log_error_q("pq_open: \"%s\" failed: %s",
pqfname, status > 0 ? strerror(status) :
"Internal error");
}
exit(exit_pq_open);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
MD5_CTX *md5ctxp = NULL;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX();
if(md5ctxp == NULL)
{
log_syserr_q("new_md5_CTX failed");
exit(exit_md5);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
if (ac > 1) {
multipleFiles = TRUE;
}
for(prod.info.seqno = seq_start ; ac > 0 ;
av++, ac--, prod.info.seqno++)
{
filename = *av;
fd = open(filename, O_RDONLY, 0);
if(fd == -1)
{
log_syserr_q("open: %s", filename);
exitCode = exit_infile;
continue;
}
if( fstat(fd, &statb) == -1)
{
log_syserr_q("fstat: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* Determine what to use for product identifier */
if (useProductID)
{
if (multipleFiles)
{
sprintf(identifier,"%s.%d", productID, prod.info.seqno);
prod.info.ident = identifier;
}
else
prod.info.ident = productID;
}
else
prod.info.ident = filename;
prod.info.sz = statb.st_size;
prod.data = NULL;
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
log_syserr_q("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
#if USE_MMAP
prod.data = mmap(0, prod.info.sz,
PROT_READ, MAP_PRIVATE, fd, 0);
if(prod.data == MAP_FAILED)
{
log_syserr_q("mmap: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: mm_md5(md5ctxp, prod.data, prod.info.sz,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
log_syserr_q("mm_md5: %s", filename);
(void) munmap(prod.data, prod.info.sz);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
log_syserr_q("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
/*
* Do the deed
*/
status = pq_insert(pq, &prod);
switch (status) {
case ENOERR:
/* no error */
if(log_is_enabled_info)
log_info_q("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug)) ;
break;
case PQUEUE_DUP:
log_error_q("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_dup;
break;
case PQUEUE_BIG:
log_error_q("Product too big for queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_infile;
break;
case ENOMEM:
log_error_q("queue full?");
exitCode = exit_system;
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
log_error_q("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
break;
}
(void) munmap(prod.data, prod.info.sz);
#else // USE_MMAP above; !USE_MMAP below
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: fd_md5(md5ctxp, fd, statb.st_size,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
log_syserr_q("xx_md5: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
if(lseek(fd, 0, SEEK_SET) == (off_t)-1)
{
log_syserr_q("rewind: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
pqeIndex = PQE_NONE;
status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex);
if(status != ENOERR) {
log_syserr_q("pqe_new: %s", filename);
exitCode = exit_infile;
}
else {
ssize_t nread = read(fd, prod.data, prod.info.sz);
(void)exitIfDone(1);
if (nread != prod.info.sz) {
log_syserr_q("read %s %u", filename, prod.info.sz);
status = EIO;
}
else {
status = pqe_insert(pq, pqeIndex);
pqeIndex = PQE_NONE;
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
log_info_q("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug)) ;
break;
case PQUEUE_DUP:
log_error_q("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_dup;
break;
case ENOMEM:
log_error_q("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
log_error_q("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
}
} /* data read into `pqeIndex` region */
if (status != ENOERR) {
(void)pqe_discard(pq, pqeIndex);
pqeIndex = PQE_NONE;
}
} /* `pqeIndex` region allocated */
#endif
(void) close(fd);
} /* input-file loop */
free_MD5_CTX(md5ctxp);
} /* code block */
exit(exitCode);
}
/*
* Reads one record's worth of data from the FIFO and writes it to a
* client-supplied buffer. Blocks until data is available.
*
* Arguments:
* shm Pointer to the shared-memory FIFO data-structure. The
* FIFO must be unlocked.
* data Pointer to the buffer into which to put data from the
* FIFO.
* sz The size of the buffer in bytes.
* nbytes Pointer to the memory location to be set to the number
* of bytes read.
* Returns:
* 0 Success. "*nbytes" is set to the number of bytes read.
* ECANCELED Operating-system failure. Error-message logged.
* EINVAL "shm" uninitialized. Error-message logged.
* EINVAL "sz" is non-positive. Error-message logged.
* EINVAL The buffer is too small for the record's data. No data
* is read. Error-message logged.
* EIO FIFO is corrupt. Error-message logged.
* Raises:
* SIGSEGV if "shm" is NULL
* SIGSEGV if "data" is NULL
* SIGSEGV if "nbytes" is NULL
* SIGABRT if "shm" is uninitialized
*/
int
shmfifo_get(
const struct shmhandle* const shm,
void* const data,
const int sz,
int* const nbytes)
{
int status;
if (sz <= 0) {
log_error_q("Non-positive number of bytes to read: %d", sz);
status = EINVAL;
}
else {
int loggedEmptyFifo = 0;
if ((status = shmfifo_lock(shm)) == 0) {
shmfifo_printmemstatus(shm);
for (status = 0; shmfifo_ll_memused(shm) == 0; ) {
if (!loggedEmptyFifo) {
log_info_q("shmfifo_get(): FIFO is empty");
loggedEmptyFifo = 1;
}
if ((status = shmfifo_wait_reader(shm)) != 0) {
break;
}
}
if (0 == status) {
struct shmbh header;
if (shmfifo_ll_memused(shm) < (int)sizeof(header)) {
log_error_q("Insufficient data for a record: "
"should be at least %d bytes; was %d bytes",
sizeof(header), shmfifo_ll_memused(shm));
shmfifo_print(shm);
status = EINVAL;
}
else {
shmfifo_ll_get(shm, &header, sizeof(header));
if (header.canary != 0xDEADBEEF) {
log_error_q("Invalid header sentinel: 0x%X",
header.canary);
status = EIO;
}
else if (shmfifo_ll_memused(shm) < header.sz) {
log_error_q("Inconsistent data-length of record: "
"expected %d bytes; encountered %d bytes",
header.sz, shmfifo_ll_memused(shm));
shmfifo_print(shm);
status = EIO;
}
else if (header.sz > sz) {
log_error_q("Client-supplied buffer too small: "
"need %d bytes; %d bytes supplied",
header.sz, sz);
shmfifo_ll_hrewind(shm);
status = EINVAL;
}
else {
shmfifo_ll_get(shm, data, header.sz);
if (loggedEmptyFifo) {
log_info_q("shmfifo_get(): "
"Got %d bytes of data from FIFO",
header.sz);
}
shmfifo_printmemstatus(shm);
if ((status = shmfifo_notify_writer(shm)) == 0) {
*nbytes = header.sz;
}
}
}
} /* FIFO has data */
int tmpStatus = shmfifo_unlock(shm);
if (status == 0)
status = tmpStatus;
} /* shared-memory FIFO locked */
}
return status;
}
/*
* Writes data to the shared-memory FIFO.
*
* Arguments:
* shm Pointer to the shared-memory FIFO data-structure.
* data Pointer to the data to be written.
* sz The amount of data to be written in bytes.
* Returns:
* 0 Success.
* E2BIG "sz" is larger than the FIFO can handle. Error-message
* logged.
* ECANCELED Operating-system failure. Error-message logged.
* EIO I/O error. Error-message logged.
* EINVAL "sz" is negative. Error-message logged.
* EINVAL "shm" uninitialized. Error-message logged.
*/
int
shmfifo_put(
const struct shmhandle* const shm,
void* const data,
const int sz)
{
int status;
if (0 > sz) {
log_error_q("Invalid size argument: %d", sz);
status = EINVAL;
}
else {
if ((status = shmfifo_lock(shm)) == 0) {
struct shmbh header;
const size_t totalBytesToWrite = sz + sizeof(header);
size_t maxSize;
shmfifo_printmemstatus(shm);
maxSize = shmfifo_ll_memused(shm) + shmfifo_ll_memfree(shm);
if (maxSize < totalBytesToWrite) {
log_error_q("Record bigger than entire FIFO: "
"record is %lu bytes; FIFO capacity is %lu bytes",
totalBytesToWrite, maxSize);
status = E2BIG;
}
else {
int loggedNoRoom = 0;
int freeSpace;
status = 0;
/*
* Wait for the FIFO to have room for the data.
*/
while ((freeSpace = shmfifo_ll_memfree(shm)) <=
totalBytesToWrite) {
if (!loggedNoRoom) {
log_error_q("No room in FIFO: "
"need %d bytes; only %d bytes available. "
"Waiting...", totalBytesToWrite, freeSpace);
loggedNoRoom = 1;
}
if ((status = shmfifo_wait_writer(shm)) != 0) {
break;
}
}
if (0 == status) {
header.sz = sz;
header.canary = 0xDEADBEEF;
shmfifo_ll_put(shm, &header, sizeof(header));
shmfifo_ll_put(shm, data, sz);
if (loggedNoRoom) {
log_info_q("shmfifo_put(): Wrote %d bytes to FIFO",
totalBytesToWrite);
}
status = shmfifo_notify_reader(shm);
}
}
int tmpStatus = shmfifo_unlock(shm);
if (status == 0)
status = tmpStatus;
} /* shared-memory FIFO locked */
} // `sz` is valid
return status;
}
int
main (int ac, char *av[])
{
char *progname = av[0];
int status;
int seq_start = 0;
stat_info *sinfo, *shead = NULL, *slast = NULL;
int statusoff=0;
/*
* Set up error logging
*/
if (log_init(progname)) {
log_syserr("Couldn't initialize logging module");
exit(1);
}
const char* pqfname = getQueuePath();
/*
* Check the environment for some options.
* May be overridden by command line switches below.
*/
{
const char *ldmpqfname = getenv ("LDMPQFNAME");
if (ldmpqfname != NULL)
pqfname = ldmpqfname;
}
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
opterr = 1;
while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF)
switch (ch)
{
case 'v':
if (!log_is_enabled_info)
(void)log_set_level(LOG_LEVEL_INFO);
break;
case 'x':
(void)log_set_level(LOG_LEVEL_DEBUG);
break;
case 'l':
if (log_set_destination(optarg)) {
log_syserr("Couldn't set logging destination to \"%s\"",
optarg);
exit(1);
}
break;
case 'q':
pqfname = optarg;
break;
case 's':
seq_start = atoi (optarg);
break;
case 'f':
feedtype = atofeedtypet (optarg);
if (feedtype == NONE)
{
fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg);
usage (progname);
}
break;
case 'S':
statusoff=1;
break;
case '?':
usage (progname);
break;
}
setQueuePath(pqfname);
ac -= optind;
av += optind;
if (ac < 1)
usage (progname);
}
/*
* register exit handler
*/
if (atexit (cleanup) != 0)
{
log_syserr ("atexit");
exit (1);
}
/*
* set up signal handlers
*/
set_sigactions ();
/*
* who am i, anyway
*/
(void) strcpy (myname, ghostname ());
/*
* open the product queue
*/
if ((status = pq_open (pqfname, PQ_DEFAULT, &pq)))
{
if (status > 0) {
log_add_syserr("\"%s\" failed", pqfname);
log_flush_error();
}
else {
log_error_q("\"%s\" failed: %s", pqfname, "Internal error");
}
exit (2);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
unsigned char *prodmmap;
MD5_CTX *md5ctxp = NULL;
int gversion;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX ();
if (md5ctxp == NULL)
{
log_syserr ("new_md5_CTX failed");
exit (6);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
prod.info.seqno = seq_start;
/*
* Open the file to be inserted and process
*/
while (ac > 0)
{
long insert_sum = 0;
long sinfo_cnt = 0;
long stat_size = 0;
filename = *av;
av++;
ac--;
log_notice_q ("open and memorymap %s\0", filename);
fd = open (filename, O_RDONLY, 0);
if (fd == -1)
{
log_syserr ("open: %s", filename);
continue;
}
if (fstat (fd, &statb) == -1)
{
log_syserr ("fstat: %s", filename);
(void) close (fd);
continue;
}
if ((prodmmap = (unsigned char *) mmap (0, statb.st_size,
PROT_READ, MAP_PRIVATE, fd,
0)) == MAP_FAILED)
{
log_syserr ("allocation failed");
}
else
{
int GRIBDONE = 0;
off_t griboff = 0;
size_t griblen = 0;
log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size);
while (!GRIBDONE)
{
log_debug("griboff %d\0", (int) griboff);
/* get offset of next grib product */
status =
get_grib_info (prodmmap, statb.st_size, &griboff, &griblen,
&gversion);
switch (status)
{
case 0:
prod.data = prodmmap + griboff;
prod.info.sz = griblen;
/*
* revised MD5 calculation...using filename
* to allow duplicate products in different files.
*/
MD5Init (md5ctxp);
MD5Update (md5ctxp, (void *)filename, strlen(filename));
/*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/
if ( prod.info.sz > 10000 )
MD5Update (md5ctxp, (void *)prod.data, 10000);
else
MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);
MD5Final (prod.info.signature, md5ctxp);
/*if (mm_md5 (md5ctxp, prod.data, prod.info.sz,
prod.info.signature) != 0)
{
log_error_q ("could not compute MD5\0");
}
else
{ */
prod.info.ident = (char *) malloc (KEYSIZE + 1);
get_gribname (gversion, prod.data, prod.info.sz,
filename, prod.info.seqno,
prod.info.ident);
/*
* Do the deed
*/
status = set_timestamp (&prod.info.arrival);
if (status != ENOERR)
{
log_add_syserr("could not set timestamp");
log_flush_error();
}
/*
* Insert the product
*/
status = pq_insert (pq, &prod);
log_info_q ("%d %s\0", status, prod.info.ident);
if ( status == ENOERR )
insert_sum += prod.info.sz;
if (! statusoff )
{
/*
* Log this status
*/
sinfo_cnt++;
sinfo = (stat_info *)malloc(sizeof(stat_info));
sinfo->insertstatus = status;
sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1);
strcpy(sinfo->prodname, prod.info.ident);
sinfo->seqno = prod.info.seqno;
sinfo->prodsz = prod.info.sz;
sinfo->next = NULL;
stat_size += strlen(sinfo->prodname);
if(shead == NULL)
{
shead = sinfo;
slast = sinfo;
}
else
{
slast->next = sinfo;
slast = sinfo;
}
}
/*}*/
griboff += griblen;
prod.info.seqno++;
break;
case -1:
GRIBDONE = 1;
break;
case -2:
log_error_q ("truncated grib file at: %d", prod.info.seqno);
GRIBDONE = 1;
break;
case -7:
log_error_q ("End sequence 7777 not found where expected: %d",
prod.info.seqno);
griboff += griblen;
log_error_q("resume looking at %d\0",griboff);
break;
default:
log_error_q ("unknown error %d\0", status);
griboff += griblen;
if (griboff >= statb.st_size)
GRIBDONE = 1;
break;
}
if (griboff >= statb.st_size)
GRIBDONE = 1;
}
log_notice_q ("munmap\0");
(void) munmap ((void *)prodmmap, statb.st_size);
if ( stat_size != 0 )
/*
* Add a status message to product queue
*/
{
char *statusmess;
log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt);
statusmess = calloc((30 * sinfo_cnt) + stat_size +
strlen(filename) + 128, sizeof(char));
if(statusmess == NULL)
{
log_syserr("could not malloc status message %ld\0",
stat_size);
}
else
{
char tmpprod[512];
sinfo = shead; slast = NULL;
status = set_timestamp(&prod.info.arrival);
/* ctime ends with \n\0" */
sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n",
filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec),
insert_sum,(long)statb.st_size);
while(sinfo != NULL)
{
sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus,
sinfo->seqno,sinfo->prodsz,sinfo->prodname);
strcat(statusmess,tmpprod);
slast = sinfo;
sinfo = sinfo->next;
free(slast->prodname);
free(slast);
}
shead = NULL;
sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno);
prod.info.ident = tmpprod;
prod.data = statusmess;
prod.info.sz = strlen(statusmess);
status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature);
status = set_timestamp(&prod.info.arrival);
status = pq_insert(pq, &prod);
if(log_is_enabled_info)
log_info_q("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug)) ;
free(statusmess);
prod.info.seqno++;
}
}
}
(void) close (fd);
}
}
exit(0);
}
