/*
* Returns the LDM proxy status for a given client failure. Logs the failure.
*
* Arguments:
* proxy The LDM proxy data-structure.
* name The name of the failed message or NULL.
* info Metadata on the data-product that couldn't be sent or
* NULL.
* Returns:
* 0 Success.
* LP_TIMEDOUT The failure was due to an RPC timeout. "log_start()"
* called iff "name" isn't NULL.
* LP_RPC_ERROR RPC error. "log_start()" called iff "name" isn't NULL.
*/
static LdmProxyStatus
getStatus(
LdmProxy* const proxy,
const char* const name,
prod_info* const info)
{
LdmProxyStatus status;
struct rpc_err rpcErr;
clnt_geterr(proxy->clnt, &rpcErr);
if (0 == rpcErr.re_status) {
status = 0;
}
else {
if (NULL != name) {
LOG_START3("%s failure to host \"%s\": %s", name, proxy->host,
clnt_errmsg(proxy->clnt));
if (NULL != info) {
LOG_ADD1("Couldn't send product: %s", s_prod_info(NULL, 0,
info, ulogIsDebug()));
}
}
status = RPC_TIMEDOUT == rpcErr.re_status
? LP_TIMEDOUT
: LP_RPC_ERROR;
}
return status;
}
/*
* Sends a single, open file to an LDM as a data-product. The number of bytes
* to be sent is specified by the data-product's metadata. The bytes start at
* the beginning of the file.
*
* Arguments:
* proxy The LDM proxy data-structure.
* fd The file-descriptor open on the file to be sent.
* info The data-product's metadata. Must be completely set.
*
* Returns:
* 0 Success.
* SYSTEM_ERROR O/S failure. "log_add()" called.
* CONNECTION_ABORTED The connection was aborted. "log_add()"
* called.
*/
static int
send_product(
LdmProxy* proxy,
int fd,
prod_info* const info)
{
int status;
product product;
product.info = *info;
product.data = mmap(NULL, info->sz, PROT_READ, MAP_PRIVATE, fd, 0);
if (MAP_FAILED == product.data) {
log_syserr_q("Couldn't memory-map file");
status = SYSTEM_ERROR;
}
else {
status = lp_send(proxy, &product);
if (LP_UNWANTED == status) {
log_notice_q("Unwanted product: %s", s_prod_info(NULL, 0, info,
log_is_enabled_debug));
status = 0;
}
(void)munmap(product.data, info->sz);
} /* file is memory-mapped */
return status;
}
/*
* The RPC dispatch routine for this program.
* Registered as a callback by svc_register() below.
* Note that only NULLPROC and NOTIFICATION rpc procs are
* handled by this program.
*/
static void
notifymeprog_5(struct svc_req *rqstp, SVCXPRT *transp)
{
prod_info notice;
switch (rqstp->rq_proc) {
case NULLPROC:
(void)svc_sendreply(transp, (xdrproc_t)xdr_void, (caddr_t)NULL);
return;
case NOTIFICATION:
(void) memset((char*)¬ice, 0, sizeof(notice));
if (!svc_getargs(transp, (xdrproc_t)xdr_prod_info,
(caddr_t)¬ice))
{
svcerr_decode(transp);
return;
}
(void)exitIfDone(0);
/*
* Update the request filter with the timestamp
* we just recieved.
* N.B.: There can still be duplicates after
* a reconnect.
*/
clss.from = notice.arrival;
timestamp_incr(&clss.from);
/*
* your code here, example just logs it
*/
log_info_q("%s", s_prod_info(NULL, 0, ¬ice,
log_is_enabled_debug));
if(!svc_sendreply(transp, (xdrproc_t)xdr_ldm_replyt,
(caddr_t) &reply))
{
svcerr_systemerr(transp);
}
(void)exitIfDone(0);
if(!svc_freeargs(transp, xdr_prod_info, (caddr_t) ¬ice)) {
log_error_q("unable to free arguments");
exit(1);
}
/* no break */
default:
svcerr_noproc(transp);
return;
}
}
static void
sender_insertProducts(void)
{
pqueue* pq;
int status = pq_open(UP7_PQ_PATHNAME, 0, &pq);
CU_ASSERT_EQUAL_FATAL(status, 0);
product prod;
prod_info* info = &prod.info;
char ident[80];
void* data = NULL;
unsigned short xsubi[3] = {(unsigned short)1234567890,
(unsigned short)9876543210,
(unsigned short)1029384756
};
info->feedtype = EXP;
info->ident = ident;
info->origin = "localhost";
(void)memset(info->signature, 0, sizeof(info->signature));
for (int i = 0; i < NUM_PRODS; i++) {
const unsigned size = MAX_PROD_SIZE*erand48(xsubi) + 0.5;
const ssize_t nbytes = snprintf(ident, sizeof(ident), "%d", i);
CU_ASSERT_TRUE_FATAL(nbytes >= 0 && nbytes < sizeof(ident));
status = set_timestamp(&info->arrival);
CU_ASSERT_EQUAL_FATAL(status, 0);
info->seqno = i;
uint32_t signet = htonl(i);
(void)memcpy(info->signature+sizeof(signaturet)-sizeof(signet), &signet,
sizeof(signet));
info->sz = size;
data = realloc(data, size);
CU_ASSERT_PTR_NOT_NULL(data);
prod.data = data;
status = pq_insert(pq, &prod);
CU_ASSERT_EQUAL_FATAL(status, 0);
char buf[LDM_INFO_MAX];
LOG_ADD1("Inserted: prodInfo=\"%s\"",
s_prod_info(buf, sizeof(buf), info, 1));
log_log(LOG_INFO);
struct timespec duration;
duration.tv_sec = 0;
duration.tv_nsec = 5000000; // 5 ms
status = nanosleep(&duration, NULL);
CU_ASSERT_EQUAL_FATAL(status, 0);
}
free(data);
status = pq_close(pq);
CU_ASSERT_EQUAL_FATAL(status, 0);
}
/*ARGSUSED*/
static int
addtostats(const prod_info *infop, const void *datap,
void *xprod, size_t size, void *notused)
{
struct timeval tv;
pq_ctimestamp(pq, &tv);
if(tvIsNone(tv))
tv = TS_ZERO;
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, infop, ulogIsDebug()));
binstats(infop, &tv);
return 0;
}
static int
doOne(const prod_info *infop, const void *datap)
{
struct product prod;
int status = ENOERR;
if(ulogIsDebug())
udebug("%s", s_prod_info(NULL, 0, infop, 1));
prod.info = *infop;
prod.data = (void *)datap; /* cast away const */
nsplit++; /* ?? Do it here on only on success ?? */
status = pq_insertNoSig(opq, &prod);
if(status == ENOERR)
{
return status; /* Normal return */
}
/* else */
if(status == PQUEUE_DUP)
{
ndups++;
if(ulogIsVerbose())
uinfo("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info,
ulogIsDebug()));
return status;
}
/* else, error */
uerror("pq_insert: %s\n", strerror(status));
return status;
}
static int
split_prod(const prod_info *infop, const void *datap,
void *xprod, size_t size, void *vp)
{
size_t *nsp = (size_t *)vp;
int ns;
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, infop, ulogIsDebug()));
ns = surf_split(infop, datap, doOne);
nprods++;
(void)kill(SIGCONT, act_pid);
if(nsp != NULL && ns >= 0)
*nsp = (size_t)ns;
return 0;
}
int main(
int ac,
char *av[]
)
{
const char* const pqfname = getQueuePath();
const char* const progname = ubasename(av[0]);
int useProductID = FALSE;
int signatureFromId = FALSE;
char *productID = NULL;
int multipleFiles = FALSE;
char identifier[KEYSIZE];
int status;
int seq_start = 0;
enum ExitCode {
exit_success = 0, /* all files inserted successfully */
exit_system = 1, /* operating-system failure */
exit_pq_open = 2, /* couldn't open product-queue */
exit_infile = 3, /* couldn't process input file */
exit_dup = 4, /* input-file already in product-queue */
exit_md5 = 6 /* couldn't initialize MD5 processing */
} exitCode = exit_success;
#if !USE_MMAP
pqeIndex = PQE_NONE;
#endif
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
(void) openulog(progname, LOG_NOTIME, LOG_LDM, "-");
(void) setulogmask(LOG_UPTO(LOG_NOTICE));
opterr = 0; /* Suppress getopt(3) error messages */
while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF)
switch (ch) {
case 'i':
signatureFromId = 1;
break;
case 'v':
(void) setulogmask(getulogmask() | LOG_MASK(LOG_INFO));
break;
case 'x':
(void) setulogmask(getulogmask() | LOG_MASK(LOG_DEBUG));
break;
case 'l':
openulog(progname, ulog_get_options(), LOG_LDM, optarg);
break;
case 'q':
setQueuePath(optarg);
break;
case 's':
seq_start = atoi(optarg);
break;
case 'f':
feedtype = atofeedtypet(optarg);
if(feedtype == NONE)
{
fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg);
usage(progname);
}
break;
case 'p':
useProductID = TRUE;
productID = optarg;
break;
case ':': {
LOG_ADD1("Option \"-%c\" requires an operand", optopt);
usage(progname);
}
/* no break */
default:
LOG_ADD1("Unknown option: \"%c\"", optopt);
usage(progname);
/* no break */
}
ac -= optind; av += optind ;
if(ac < 1) usage(progname);
}
/*
* register exit handler
*/
if(atexit(cleanup) != 0)
{
serror("atexit");
exit(exit_system);
}
/*
* set up signal handlers
*/
set_sigactions();
/*
* who am i, anyway
*/
(void) strncpy(myname, ghostname(), sizeof(myname));
myname[sizeof(myname)-1] = 0;
/*
* open the product queue
*/
if(status = pq_open(pqfname, PQ_DEFAULT, &pq))
{
if (PQ_CORRUPT == status) {
uerror("The product-queue \"%s\" is inconsistent\n",
pqfname);
}
else {
uerror("pq_open: \"%s\" failed: %s",
pqfname, status > 0 ? strerror(status) :
"Internal error");
}
exit(exit_pq_open);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
MD5_CTX *md5ctxp = NULL;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX();
if(md5ctxp == NULL)
{
serror("new_md5_CTX failed");
exit(exit_md5);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
if (ac > 1) {
multipleFiles = TRUE;
}
for(prod.info.seqno = seq_start ; ac > 0 ;
av++, ac--, prod.info.seqno++)
{
filename = *av;
fd = open(filename, O_RDONLY, 0);
if(fd == -1)
{
serror("open: %s", filename);
exitCode = exit_infile;
continue;
}
if( fstat(fd, &statb) == -1)
{
serror("fstat: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* Determine what to use for product identifier */
if (useProductID)
{
if (multipleFiles)
{
sprintf(identifier,"%s.%d", productID, prod.info.seqno);
prod.info.ident = identifier;
}
else
prod.info.ident = productID;
}
else
prod.info.ident = filename;
prod.info.sz = statb.st_size;
prod.data = NULL;
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
serror("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
#if USE_MMAP
prod.data = mmap(0, prod.info.sz,
PROT_READ, MAP_PRIVATE, fd, 0);
if(prod.data == NULL)
{
serror("mmap: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: mm_md5(md5ctxp, prod.data, prod.info.sz,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("mm_md5: %s", filename);
(void) munmap(prod.data, prod.info.sz);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
serror("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
/*
* Do the deed
*/
status = pq_insert(pq, &prod);
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_dup;
break;
case PQUEUE_BIG:
uerror("Product too big for queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_infile;
break;
case ENOMEM:
uerror("queue full?");
exitCode = exit_system;
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
break;
}
(void) munmap(prod.data, prod.info.sz);
#else // USE_MMAP above; !USE_MMAP below
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: fd_md5(md5ctxp, fd, statb.st_size,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("xx_md5: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
if(lseek(fd, 0, SEEK_SET) == (off_t)-1)
{
serror("rewind: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
pqeIndex = PQE_NONE;
status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex);
if(status != ENOERR) {
serror("pqe_new: %s", filename);
exitCode = exit_infile;
}
else {
ssize_t nread = read(fd, prod.data, prod.info.sz);
(void)exitIfDone(1);
if (nread != prod.info.sz) {
serror("read %s %u", filename, prod.info.sz);
status = EIO;
}
else {
status = pqe_insert(pq, pqeIndex);
pqeIndex = PQE_NONE;
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_dup;
break;
case ENOMEM:
uerror("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
}
} /* data read into `pqeIndex` region */
if (status != ENOERR) {
(void)pqe_discard(pq, pqeIndex);
pqeIndex = PQE_NONE;
}
} /* `pqeIndex` region allocated */
#endif
(void) close(fd);
} /* input-file loop */
free_MD5_CTX(md5ctxp);
} /* code block */
exit(exitCode);
}
/*
* Arguments:
* arrival Data-product creation-time. IGNORED. The creation-time
* will be set by this function according to the system clock
* just prior to inserting the data-product into the product-
* queue.
* seqno Sequence number.
* ident Product-identifier.
* len Size of data-portion of data-product in bytes.
* buf Pointer to data-portion of data-product.
*/
void
toClients(timestampt arrival,
unsigned seqno,
const char *ident,
unsigned len,
const char *buf)
{
static struct product prod;
int status;
char *result;
MD5Init(md5ctxp);
/*
* If user has not disabled skipLeadingCtlString with -5 option,
* and the product contains the correct leading control string for
* a wmo product, then skip that control string in calculating the
* checksum.
*/
if (skipLeadingCtlString && ((result = wmo_prod(buf)) != 0))
{
MD5Update(md5ctxp, (const unsigned char *)result, len-(result-buf));
#if DEBUG
log_info("WMO prod: Skipping %d chars\n", result-buf);
#endif
}
else /* calculate checksum on entire product */
{
MD5Update(md5ctxp, (const unsigned char *)buf, len);
#if DEBUG
log_info("not a WMO Prod\n");
#endif
}
MD5Final((unsigned char*)prod.info.signature, md5ctxp);
prod.info.origin = myname;
prod.info.feedtype = feedtype;
prod.info.seqno = seqno;
prod.info.ident = (char *)ident; /* cast away const */
prod.info.sz = len;
prod.data = (void *)buf; /* cast away const */
if(((strncmp(prod.info.ident,"SDUS2",5) == 0) ||
(strncmp(prod.info.ident,"SDUS3",5) == 0) ||
(strncmp(prod.info.ident,"SDUS5",5) == 0) ||
(strncmp(prod.info.ident,"SDUS7",5) == 0)) && (useNex == 1))
{
prod.info.feedtype = NEXRAD;
}
if(log_is_enabled_info)
log_info("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug));
if(pq == NULL) /* if we are "feedtest", do nothing else */
return;
set_timestamp(&prod.info.arrival);
status = pq_insert(pq, &prod);
if(status == ENOERR)
return; /* Normal return */
/* else */
if(status == PQUEUE_DUP)
{
ndups++;
log_info("Product already in queue");
return;
}
/* else, error */
if (status > 0) {
log_errno(status, "pq_insert");
}
else {
log_error("pq_insert: Internal error");
}
exit(1); /* ??? */
}
int main(
int ac,
char *av[]
)
{
char *progname = av[0];
char *logfname;
int useProductID = FALSE;
int signatureFromId = FALSE;
char *productID = NULL;
int multipleFiles = FALSE;
char identifier[KEYSIZE];
int status;
int seq_start = 0;
enum ExitCode {
exit_success = 0,
exit_system = 1, /* operating-system failure */
exit_pq_open = 2, /* couldn't open product-queue */
exit_infile = 3, /* couldn't process input file */
exit_md5 = 6 /* couldn't initialize MD5 processing */
} exitCode = exit_success;
logfname = "-";
/*
* Check the environment for some options.
* May be overridden by command line switches below.
*/
{
const char *ldmpqfname = getenv("LDMPQFNAME");
if(ldmpqfname != NULL)
pqfname = ldmpqfname;
}
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) |
LOG_MASK(LOG_NOTICE));
opterr = 1;
while ((ch = getopt(ac, av, "ivxl:q:f:s:p:")) != EOF)
switch (ch) {
case 'i':
signatureFromId = 1;
break;
case 'v':
logmask |= LOG_MASK(LOG_INFO);
break;
case 'x':
logmask |= LOG_MASK(LOG_DEBUG);
break;
case 'l':
logfname = optarg;
break;
case 'q':
pqfname = optarg;
break;
case 's':
seq_start = atoi(optarg);
break;
case 'f':
feedtype = atofeedtypet(optarg);
if(feedtype == NONE)
{
fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg);
usage(progname);
}
break;
case 'p':
useProductID = TRUE;
productID = optarg;
break;
case '?':
usage(progname);
break;
}
ac -= optind; av += optind ;
if(ac < 1) usage(progname);
(void) setulogmask(logmask);
}
/*
* Set up error logging
*/
(void) openulog(ubasename(progname), LOG_NOTIME, LOG_LDM, logfname);
/*
* register exit handler
*/
if(atexit(cleanup) != 0)
{
serror("atexit");
exit(exit_system);
}
/*
* set up signal handlers
*/
set_sigactions();
/*
* who am i, anyway
*/
(void) strcpy(myname, ghostname());
/*
* open the product queue
*/
if(status = pq_open(pqfname, PQ_DEFAULT, &pq))
{
if (PQ_CORRUPT == status) {
uerror("The product-queue \"%s\" is inconsistent\n",
pqfname);
}
else {
uerror("pq_open: \"%s\" failed: %s",
pqfname, status > 0 ? strerror(status) :
"Internal error");
}
exit(exit_pq_open);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
MD5_CTX *md5ctxp = NULL;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX();
if(md5ctxp == NULL)
{
serror("new_md5_CTX failed");
exit(exit_md5);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
if (ac > 1) {
multipleFiles = TRUE;
}
for(prod.info.seqno = seq_start ; ac > 0 ;
av++, ac--, prod.info.seqno++)
{
filename = *av;
fd = open(filename, O_RDONLY, 0);
if(fd == -1)
{
serror("open: %s", filename);
exitCode = exit_infile;
continue;
}
if( fstat(fd, &statb) == -1)
{
serror("fstat: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* Determine what to use for product identifier */
if (useProductID)
{
if (multipleFiles)
{
sprintf(identifier,"%s.%d", productID, prod.info.seqno);
prod.info.ident = identifier;
}
else
prod.info.ident = productID;
}
else
prod.info.ident = filename;
prod.info.sz = statb.st_size;
prod.data = NULL;
#ifndef NO_MMAP
prod.data = mmap(0, prod.info.sz,
PROT_READ, MAP_PRIVATE, fd, 0);
if(prod.data == NULL)
{
serror("mmap: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: mm_md5(md5ctxp, prod.data, prod.info.sz,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("mm_md5: %s", filename);
(void) munmap(prod.data, prod.info.sz);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
serror("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
/*
* Do the deed
*/
status = pq_insert(pq, &prod);
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
break;
case ENOMEM:
uerror("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
break;
}
(void) munmap(prod.data, prod.info.sz);
#else /*!NO_MMAP*/
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: fd_md5(md5ctxp, fd, statb.st_size,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("fd_md5: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
if(lseek(fd, 0, SEEK_SET) == (off_t)-1)
{
serror("rewind: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
index = PQE_NONE;
status = pqe_new(pq, &prod.info, &prod.data, &index);
if(status != ENOERR) {
serror("pqe_new: %s", filename);
exitCode = exit_infile;
}
else {
ssize_t nread = read(fd, prod.data, prod.info.sz);
(void)exitIfDone(1);
if (nread != prod.info.sz) {
serror("read %s %u", filename, prod.info.sz);
status = EIO;
}
else {
status = pqe_insert(pq, index);
index = PQE_NONE;
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
break;
case ENOMEM:
uerror("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
}
} /* data read into "index" region */
if (status != ENOERR) {
(void)pqe_discard(pq, index);
index = PQE_NONE;
}
} /* "index" region allocated */
#endif /*!NO_MMAP*/
(void) close(fd);
} /* input-file loop */
free_MD5_CTX(md5ctxp);
} /* code block */
exit(exitCode);
}
int
main (int ac, char *av[])
{
char *progname = av[0];
int status;
int seq_start = 0;
stat_info *sinfo, *shead = NULL, *slast = NULL;
int statusoff=0;
/*
* Set up error logging
*/
if (log_init(progname)) {
log_syserr("Couldn't initialize logging module");
exit(1);
}
const char* pqfname = getQueuePath();
/*
* Check the environment for some options.
* May be overridden by command line switches below.
*/
{
const char *ldmpqfname = getenv ("LDMPQFNAME");
if (ldmpqfname != NULL)
pqfname = ldmpqfname;
}
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
opterr = 1;
while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF)
switch (ch)
{
case 'v':
if (!log_is_enabled_info)
(void)log_set_level(LOG_LEVEL_INFO);
break;
case 'x':
(void)log_set_level(LOG_LEVEL_DEBUG);
break;
case 'l':
if (log_set_destination(optarg)) {
log_syserr("Couldn't set logging destination to \"%s\"",
optarg);
exit(1);
}
break;
case 'q':
pqfname = optarg;
break;
case 's':
seq_start = atoi (optarg);
break;
case 'f':
feedtype = atofeedtypet (optarg);
if (feedtype == NONE)
{
fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg);
usage (progname);
}
break;
case 'S':
statusoff=1;
break;
case '?':
usage (progname);
break;
}
setQueuePath(pqfname);
ac -= optind;
av += optind;
if (ac < 1)
usage (progname);
}
/*
* register exit handler
*/
if (atexit (cleanup) != 0)
{
log_syserr ("atexit");
exit (1);
}
/*
* set up signal handlers
*/
set_sigactions ();
/*
* who am i, anyway
*/
(void) strcpy (myname, ghostname ());
/*
* open the product queue
*/
if ((status = pq_open (pqfname, PQ_DEFAULT, &pq)))
{
if (status > 0) {
log_add_syserr("\"%s\" failed", pqfname);
log_flush_error();
}
else {
log_error_q("\"%s\" failed: %s", pqfname, "Internal error");
}
exit (2);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
unsigned char *prodmmap;
MD5_CTX *md5ctxp = NULL;
int gversion;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX ();
if (md5ctxp == NULL)
{
log_syserr ("new_md5_CTX failed");
exit (6);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
prod.info.seqno = seq_start;
/*
* Open the file to be inserted and process
*/
while (ac > 0)
{
long insert_sum = 0;
long sinfo_cnt = 0;
long stat_size = 0;
filename = *av;
av++;
ac--;
log_notice_q ("open and memorymap %s\0", filename);
fd = open (filename, O_RDONLY, 0);
if (fd == -1)
{
log_syserr ("open: %s", filename);
continue;
}
if (fstat (fd, &statb) == -1)
{
log_syserr ("fstat: %s", filename);
(void) close (fd);
continue;
}
if ((prodmmap = (unsigned char *) mmap (0, statb.st_size,
PROT_READ, MAP_PRIVATE, fd,
0)) == MAP_FAILED)
{
log_syserr ("allocation failed");
}
else
{
int GRIBDONE = 0;
off_t griboff = 0;
size_t griblen = 0;
log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size);
while (!GRIBDONE)
{
log_debug("griboff %d\0", (int) griboff);
/* get offset of next grib product */
status =
get_grib_info (prodmmap, statb.st_size, &griboff, &griblen,
&gversion);
switch (status)
{
case 0:
prod.data = prodmmap + griboff;
prod.info.sz = griblen;
/*
* revised MD5 calculation...using filename
* to allow duplicate products in different files.
*/
MD5Init (md5ctxp);
MD5Update (md5ctxp, (void *)filename, strlen(filename));
/*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/
if ( prod.info.sz > 10000 )
MD5Update (md5ctxp, (void *)prod.data, 10000);
else
MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);
MD5Final (prod.info.signature, md5ctxp);
/*if (mm_md5 (md5ctxp, prod.data, prod.info.sz,
prod.info.signature) != 0)
{
log_error_q ("could not compute MD5\0");
}
else
{ */
prod.info.ident = (char *) malloc (KEYSIZE + 1);
get_gribname (gversion, prod.data, prod.info.sz,
filename, prod.info.seqno,
prod.info.ident);
/*
* Do the deed
*/
status = set_timestamp (&prod.info.arrival);
if (status != ENOERR)
{
log_add_syserr("could not set timestamp");
log_flush_error();
}
/*
* Insert the product
*/
status = pq_insert (pq, &prod);
log_info_q ("%d %s\0", status, prod.info.ident);
if ( status == ENOERR )
insert_sum += prod.info.sz;
if (! statusoff )
{
/*
* Log this status
*/
sinfo_cnt++;
sinfo = (stat_info *)malloc(sizeof(stat_info));
sinfo->insertstatus = status;
sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1);
strcpy(sinfo->prodname, prod.info.ident);
sinfo->seqno = prod.info.seqno;
sinfo->prodsz = prod.info.sz;
sinfo->next = NULL;
stat_size += strlen(sinfo->prodname);
if(shead == NULL)
{
shead = sinfo;
slast = sinfo;
}
else
{
slast->next = sinfo;
slast = sinfo;
}
}
/*}*/
griboff += griblen;
prod.info.seqno++;
break;
case -1:
GRIBDONE = 1;
break;
case -2:
log_error_q ("truncated grib file at: %d", prod.info.seqno);
GRIBDONE = 1;
break;
case -7:
log_error_q ("End sequence 7777 not found where expected: %d",
prod.info.seqno);
griboff += griblen;
log_error_q("resume looking at %d\0",griboff);
break;
default:
log_error_q ("unknown error %d\0", status);
griboff += griblen;
if (griboff >= statb.st_size)
GRIBDONE = 1;
break;
}
if (griboff >= statb.st_size)
GRIBDONE = 1;
}
log_notice_q ("munmap\0");
(void) munmap ((void *)prodmmap, statb.st_size);
if ( stat_size != 0 )
/*
* Add a status message to product queue
*/
{
char *statusmess;
log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt);
statusmess = calloc((30 * sinfo_cnt) + stat_size +
strlen(filename) + 128, sizeof(char));
if(statusmess == NULL)
{
log_syserr("could not malloc status message %ld\0",
stat_size);
}
else
{
char tmpprod[512];
sinfo = shead; slast = NULL;
status = set_timestamp(&prod.info.arrival);
/* ctime ends with \n\0" */
sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n",
filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec),
insert_sum,(long)statb.st_size);
while(sinfo != NULL)
{
sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus,
sinfo->seqno,sinfo->prodsz,sinfo->prodname);
strcat(statusmess,tmpprod);
slast = sinfo;
sinfo = sinfo->next;
free(slast->prodname);
free(slast);
}
shead = NULL;
sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno);
prod.info.ident = tmpprod;
prod.data = statusmess;
prod.info.sz = strlen(statusmess);
status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature);
status = set_timestamp(&prod.info.arrival);
status = pq_insert(pq, &prod);
if(log_is_enabled_info)
log_info_q("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug)) ;
free(statusmess);
prod.info.seqno++;
}
}
}
(void) close (fd);
}
}
exit(0);
}
/*
* Send a product from file-descriptor to clnt using LDM-6 protocol.
*/
static void
send_product_6(CLIENT *clnt, int fd, prod_info *infop)
{
unsigned size = infop->sz;
if (size <= max_hereis) {
/*
* The file is small enough to be sent in a single HEREIS message.
*/
void* buf = (char*)malloc(size);
udebug("Sending file via HEREIS");
if (NULL == buf) {
serror("Couldn't allocate %u bytes for product data", size);
}
else {
ssize_t nread = read(fd, buf, size);
if(nread != size) {
serror("Couldn't read %u bytes of data", size);
}
else {
product product;
product.info = *infop;
product.data = buf;
if (NULL == hereis_6(&product, clnt)) {
uerror("%s: HEREIS_6 failure: %s",
remote, clnt_errmsg(clnt));
}
}
free(buf);
}
}
else {
/*
* The file is so large that it must be sent via COMINGSOON/BLKDATA
* messages.
*/
comingsoon_reply_t* reply;
comingsoon_args soonArg;
udebug("Sending file via COMINGSOON/BLKDATA");
soonArg.infop = infop;
soonArg.pktsz = size;
reply = comingsoon_6(&soonArg, clnt);
if (NULL == reply) {
uerror("%s: COMINGSOON_6 failure: %s", remote, clnt_errmsg(clnt));
}
else {
if (DONT_SEND == *reply) {
if (ulogIsVerbose() || ulogIsDebug())
uinfo("Downstream LDM says don't send: %s",
s_prod_info(NULL, 0, infop, ulogIsDebug()));
}
else if (0 != *reply) {
uwarn("Unexpected reply (%s) from downstream LDM: %s",
s_prod_info(NULL, 0, infop, ulogIsDebug()));
}
else {
void* buf = (char*)malloc(size);
if (NULL == buf) {
serror("Couldn't allocate %u bytes for product data",
size);
}
else {
ssize_t nread = read(fd, buf, size);
if(nread != size) {
serror("Couldn't read %u bytes of data", size);
}
else {
datapkt packet;
packet.signaturep = (signaturet*)&infop->signature;
packet.pktnum = 0;
packet.data.dbuf_len = size;
packet.data.dbuf_val = buf;
if (NULL == blkdata_6(&packet, clnt)) {
uerror("%s: BLKDATA_6 failure: %s",
remote, clnt_errmsg(clnt));
}
}
free(buf);
}
}
}
}
}