int main(
int ac,
char *av[]
)
{
char *progname = av[0];
char *logfname;
int useProductID = FALSE;
int signatureFromId = FALSE;
char *productID = NULL;
int multipleFiles = FALSE;
char identifier[KEYSIZE];
int status;
int seq_start = 0;
enum ExitCode {
exit_success = 0,
exit_system = 1, /* operating-system failure */
exit_pq_open = 2, /* couldn't open product-queue */
exit_infile = 3, /* couldn't process input file */
exit_md5 = 6 /* couldn't initialize MD5 processing */
} exitCode = exit_success;
logfname = "-";
/*
* Check the environment for some options.
* May be overridden by command line switches below.
*/
{
const char *ldmpqfname = getenv("LDMPQFNAME");
if(ldmpqfname != NULL)
pqfname = ldmpqfname;
}
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
int logmask = (LOG_MASK(LOG_ERR) | LOG_MASK(LOG_WARNING) |
LOG_MASK(LOG_NOTICE));
opterr = 1;
while ((ch = getopt(ac, av, "ivxl:q:f:s:p:")) != EOF)
switch (ch) {
case 'i':
signatureFromId = 1;
break;
case 'v':
logmask |= LOG_MASK(LOG_INFO);
break;
case 'x':
logmask |= LOG_MASK(LOG_DEBUG);
break;
case 'l':
logfname = optarg;
break;
case 'q':
pqfname = optarg;
break;
case 's':
seq_start = atoi(optarg);
break;
case 'f':
feedtype = atofeedtypet(optarg);
if(feedtype == NONE)
{
fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg);
usage(progname);
}
break;
case 'p':
useProductID = TRUE;
productID = optarg;
break;
case '?':
usage(progname);
break;
}
ac -= optind; av += optind ;
if(ac < 1) usage(progname);
(void) setulogmask(logmask);
}
/*
* Set up error logging
*/
(void) openulog(ubasename(progname), LOG_NOTIME, LOG_LDM, logfname);
/*
* register exit handler
*/
if(atexit(cleanup) != 0)
{
serror("atexit");
exit(exit_system);
}
/*
* set up signal handlers
*/
set_sigactions();
/*
* who am i, anyway
*/
(void) strcpy(myname, ghostname());
/*
* open the product queue
*/
if(status = pq_open(pqfname, PQ_DEFAULT, &pq))
{
if (PQ_CORRUPT == status) {
uerror("The product-queue \"%s\" is inconsistent\n",
pqfname);
}
else {
uerror("pq_open: \"%s\" failed: %s",
pqfname, status > 0 ? strerror(status) :
"Internal error");
}
exit(exit_pq_open);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
MD5_CTX *md5ctxp = NULL;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX();
if(md5ctxp == NULL)
{
serror("new_md5_CTX failed");
exit(exit_md5);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
if (ac > 1) {
multipleFiles = TRUE;
}
for(prod.info.seqno = seq_start ; ac > 0 ;
av++, ac--, prod.info.seqno++)
{
filename = *av;
fd = open(filename, O_RDONLY, 0);
if(fd == -1)
{
serror("open: %s", filename);
exitCode = exit_infile;
continue;
}
if( fstat(fd, &statb) == -1)
{
serror("fstat: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* Determine what to use for product identifier */
if (useProductID)
{
if (multipleFiles)
{
sprintf(identifier,"%s.%d", productID, prod.info.seqno);
prod.info.ident = identifier;
}
else
prod.info.ident = productID;
}
else
prod.info.ident = filename;
prod.info.sz = statb.st_size;
prod.data = NULL;
#ifndef NO_MMAP
prod.data = mmap(0, prod.info.sz,
PROT_READ, MAP_PRIVATE, fd, 0);
if(prod.data == NULL)
{
serror("mmap: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: mm_md5(md5ctxp, prod.data, prod.info.sz,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("mm_md5: %s", filename);
(void) munmap(prod.data, prod.info.sz);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
serror("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
/*
* Do the deed
*/
status = pq_insert(pq, &prod);
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
break;
case ENOMEM:
uerror("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
break;
}
(void) munmap(prod.data, prod.info.sz);
#else /*!NO_MMAP*/
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: fd_md5(md5ctxp, fd, statb.st_size,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("fd_md5: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
if(lseek(fd, 0, SEEK_SET) == (off_t)-1)
{
serror("rewind: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
index = PQE_NONE;
status = pqe_new(pq, &prod.info, &prod.data, &index);
if(status != ENOERR) {
serror("pqe_new: %s", filename);
exitCode = exit_infile;
}
else {
ssize_t nread = read(fd, prod.data, prod.info.sz);
(void)exitIfDone(1);
if (nread != prod.info.sz) {
serror("read %s %u", filename, prod.info.sz);
status = EIO;
}
else {
status = pqe_insert(pq, index);
index = PQE_NONE;
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
break;
case ENOMEM:
uerror("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
}
} /* data read into "index" region */
if (status != ENOERR) {
(void)pqe_discard(pq, index);
index = PQE_NONE;
}
} /* "index" region allocated */
#endif /*!NO_MMAP*/
(void) close(fd);
} /* input-file loop */
free_MD5_CTX(md5ctxp);
} /* code block */
exit(exitCode);
}
int main(
int ac,
char *av[]
)
{
const char* const pqfname = getQueuePath();
const char* const progname = ubasename(av[0]);
int useProductID = FALSE;
int signatureFromId = FALSE;
char *productID = NULL;
int multipleFiles = FALSE;
char identifier[KEYSIZE];
int status;
int seq_start = 0;
enum ExitCode {
exit_success = 0, /* all files inserted successfully */
exit_system = 1, /* operating-system failure */
exit_pq_open = 2, /* couldn't open product-queue */
exit_infile = 3, /* couldn't process input file */
exit_dup = 4, /* input-file already in product-queue */
exit_md5 = 6 /* couldn't initialize MD5 processing */
} exitCode = exit_success;
#if !USE_MMAP
pqeIndex = PQE_NONE;
#endif
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
(void) openulog(progname, LOG_NOTIME, LOG_LDM, "-");
(void) setulogmask(LOG_UPTO(LOG_NOTICE));
opterr = 0; /* Suppress getopt(3) error messages */
while ((ch = getopt(ac, av, ":ivxl:q:f:s:p:")) != EOF)
switch (ch) {
case 'i':
signatureFromId = 1;
break;
case 'v':
(void) setulogmask(getulogmask() | LOG_MASK(LOG_INFO));
break;
case 'x':
(void) setulogmask(getulogmask() | LOG_MASK(LOG_DEBUG));
break;
case 'l':
openulog(progname, ulog_get_options(), LOG_LDM, optarg);
break;
case 'q':
setQueuePath(optarg);
break;
case 's':
seq_start = atoi(optarg);
break;
case 'f':
feedtype = atofeedtypet(optarg);
if(feedtype == NONE)
{
fprintf(stderr, "Unknown feedtype \"%s\"\n", optarg);
usage(progname);
}
break;
case 'p':
useProductID = TRUE;
productID = optarg;
break;
case ':': {
LOG_ADD1("Option \"-%c\" requires an operand", optopt);
usage(progname);
}
/* no break */
default:
LOG_ADD1("Unknown option: \"%c\"", optopt);
usage(progname);
/* no break */
}
ac -= optind; av += optind ;
if(ac < 1) usage(progname);
}
/*
* register exit handler
*/
if(atexit(cleanup) != 0)
{
serror("atexit");
exit(exit_system);
}
/*
* set up signal handlers
*/
set_sigactions();
/*
* who am i, anyway
*/
(void) strncpy(myname, ghostname(), sizeof(myname));
myname[sizeof(myname)-1] = 0;
/*
* open the product queue
*/
if(status = pq_open(pqfname, PQ_DEFAULT, &pq))
{
if (PQ_CORRUPT == status) {
uerror("The product-queue \"%s\" is inconsistent\n",
pqfname);
}
else {
uerror("pq_open: \"%s\" failed: %s",
pqfname, status > 0 ? strerror(status) :
"Internal error");
}
exit(exit_pq_open);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
MD5_CTX *md5ctxp = NULL;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX();
if(md5ctxp == NULL)
{
serror("new_md5_CTX failed");
exit(exit_md5);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
if (ac > 1) {
multipleFiles = TRUE;
}
for(prod.info.seqno = seq_start ; ac > 0 ;
av++, ac--, prod.info.seqno++)
{
filename = *av;
fd = open(filename, O_RDONLY, 0);
if(fd == -1)
{
serror("open: %s", filename);
exitCode = exit_infile;
continue;
}
if( fstat(fd, &statb) == -1)
{
serror("fstat: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* Determine what to use for product identifier */
if (useProductID)
{
if (multipleFiles)
{
sprintf(identifier,"%s.%d", productID, prod.info.seqno);
prod.info.ident = identifier;
}
else
prod.info.ident = productID;
}
else
prod.info.ident = filename;
prod.info.sz = statb.st_size;
prod.data = NULL;
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
serror("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
#if USE_MMAP
prod.data = mmap(0, prod.info.sz,
PROT_READ, MAP_PRIVATE, fd, 0);
if(prod.data == NULL)
{
serror("mmap: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: mm_md5(md5ctxp, prod.data, prod.info.sz,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("mm_md5: %s", filename);
(void) munmap(prod.data, prod.info.sz);
(void) close(fd);
exitCode = exit_infile;
continue;
}
/* These members, and seqno, vary over the loop. */
status = set_timestamp(&prod.info.arrival);
if(status != ENOERR) {
serror("set_timestamp: %s, filename");
exitCode = exit_infile;
continue;
}
/*
* Do the deed
*/
status = pq_insert(pq, &prod);
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_dup;
break;
case PQUEUE_BIG:
uerror("Product too big for queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_infile;
break;
case ENOMEM:
uerror("queue full?");
exitCode = exit_system;
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
break;
}
(void) munmap(prod.data, prod.info.sz);
#else // USE_MMAP above; !USE_MMAP below
status =
signatureFromId
? mm_md5(md5ctxp, prod.info.ident,
strlen(prod.info.ident), prod.info.signature)
: fd_md5(md5ctxp, fd, statb.st_size,
prod.info.signature);
(void)exitIfDone(1);
if (status != 0) {
serror("xx_md5: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
if(lseek(fd, 0, SEEK_SET) == (off_t)-1)
{
serror("rewind: %s", filename);
(void) close(fd);
exitCode = exit_infile;
continue;
}
pqeIndex = PQE_NONE;
status = pqe_new(pq, &prod.info, &prod.data, &pqeIndex);
if(status != ENOERR) {
serror("pqe_new: %s", filename);
exitCode = exit_infile;
}
else {
ssize_t nread = read(fd, prod.data, prod.info.sz);
(void)exitIfDone(1);
if (nread != prod.info.sz) {
serror("read %s %u", filename, prod.info.sz);
status = EIO;
}
else {
status = pqe_insert(pq, pqeIndex);
pqeIndex = PQE_NONE;
switch (status) {
case ENOERR:
/* no error */
if(ulogIsVerbose())
uinfo("%s", s_prod_info(NULL, 0, &prod.info,
ulogIsDebug())) ;
break;
case PQUEUE_DUP:
uerror("Product already in queue: %s",
s_prod_info(NULL, 0, &prod.info, 1));
exitCode = exit_dup;
break;
case ENOMEM:
uerror("queue full?");
break;
case EINTR:
#if defined(EDEADLOCK) && EDEADLOCK != EDEADLK
case EDEADLOCK:
/*FALLTHROUGH*/
#endif
case EDEADLK:
/* TODO: retry ? */
/*FALLTHROUGH*/
default:
uerror("pq_insert: %s", status > 0
? strerror(status) : "Internal error");
}
} /* data read into `pqeIndex` region */
if (status != ENOERR) {
(void)pqe_discard(pq, pqeIndex);
pqeIndex = PQE_NONE;
}
} /* `pqeIndex` region allocated */
#endif
(void) close(fd);
} /* input-file loop */
free_MD5_CTX(md5ctxp);
} /* code block */
exit(exitCode);
}
int
main (int ac, char *av[])
{
char *progname = av[0];
int status;
int seq_start = 0;
stat_info *sinfo, *shead = NULL, *slast = NULL;
int statusoff=0;
/*
* Set up error logging
*/
if (log_init(progname)) {
log_syserr("Couldn't initialize logging module");
exit(1);
}
const char* pqfname = getQueuePath();
/*
* Check the environment for some options.
* May be overridden by command line switches below.
*/
{
const char *ldmpqfname = getenv ("LDMPQFNAME");
if (ldmpqfname != NULL)
pqfname = ldmpqfname;
}
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
opterr = 1;
while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF)
switch (ch)
{
case 'v':
if (!log_is_enabled_info)
(void)log_set_level(LOG_LEVEL_INFO);
break;
case 'x':
(void)log_set_level(LOG_LEVEL_DEBUG);
break;
case 'l':
if (log_set_destination(optarg)) {
log_syserr("Couldn't set logging destination to \"%s\"",
optarg);
exit(1);
}
break;
case 'q':
pqfname = optarg;
break;
case 's':
seq_start = atoi (optarg);
break;
case 'f':
feedtype = atofeedtypet (optarg);
if (feedtype == NONE)
{
fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg);
usage (progname);
}
break;
case 'S':
statusoff=1;
break;
case '?':
usage (progname);
break;
}
setQueuePath(pqfname);
ac -= optind;
av += optind;
if (ac < 1)
usage (progname);
}
/*
* register exit handler
*/
if (atexit (cleanup) != 0)
{
log_syserr ("atexit");
exit (1);
}
/*
* set up signal handlers
*/
set_sigactions ();
/*
* who am i, anyway
*/
(void) strcpy (myname, ghostname ());
/*
* open the product queue
*/
if ((status = pq_open (pqfname, PQ_DEFAULT, &pq)))
{
if (status > 0) {
log_add_syserr("\"%s\" failed", pqfname);
log_flush_error();
}
else {
log_error_q("\"%s\" failed: %s", pqfname, "Internal error");
}
exit (2);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
unsigned char *prodmmap;
MD5_CTX *md5ctxp = NULL;
int gversion;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX ();
if (md5ctxp == NULL)
{
log_syserr ("new_md5_CTX failed");
exit (6);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
prod.info.seqno = seq_start;
/*
* Open the file to be inserted and process
*/
while (ac > 0)
{
long insert_sum = 0;
long sinfo_cnt = 0;
long stat_size = 0;
filename = *av;
av++;
ac--;
log_notice_q ("open and memorymap %s\0", filename);
fd = open (filename, O_RDONLY, 0);
if (fd == -1)
{
log_syserr ("open: %s", filename);
continue;
}
if (fstat (fd, &statb) == -1)
{
log_syserr ("fstat: %s", filename);
(void) close (fd);
continue;
}
if ((prodmmap = (unsigned char *) mmap (0, statb.st_size,
PROT_READ, MAP_PRIVATE, fd,
0)) == MAP_FAILED)
{
log_syserr ("allocation failed");
}
else
{
int GRIBDONE = 0;
off_t griboff = 0;
size_t griblen = 0;
log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size);
while (!GRIBDONE)
{
log_debug("griboff %d\0", (int) griboff);
/* get offset of next grib product */
status =
get_grib_info (prodmmap, statb.st_size, &griboff, &griblen,
&gversion);
switch (status)
{
case 0:
prod.data = prodmmap + griboff;
prod.info.sz = griblen;
/*
* revised MD5 calculation...using filename
* to allow duplicate products in different files.
*/
MD5Init (md5ctxp);
MD5Update (md5ctxp, (void *)filename, strlen(filename));
/*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/
if ( prod.info.sz > 10000 )
MD5Update (md5ctxp, (void *)prod.data, 10000);
else
MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);
MD5Final (prod.info.signature, md5ctxp);
/*if (mm_md5 (md5ctxp, prod.data, prod.info.sz,
prod.info.signature) != 0)
{
log_error_q ("could not compute MD5\0");
}
else
{ */
prod.info.ident = (char *) malloc (KEYSIZE + 1);
get_gribname (gversion, prod.data, prod.info.sz,
filename, prod.info.seqno,
prod.info.ident);
/*
* Do the deed
*/
status = set_timestamp (&prod.info.arrival);
if (status != ENOERR)
{
log_add_syserr("could not set timestamp");
log_flush_error();
}
/*
* Insert the product
*/
status = pq_insert (pq, &prod);
log_info_q ("%d %s\0", status, prod.info.ident);
if ( status == ENOERR )
insert_sum += prod.info.sz;
if (! statusoff )
{
/*
* Log this status
*/
sinfo_cnt++;
sinfo = (stat_info *)malloc(sizeof(stat_info));
sinfo->insertstatus = status;
sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1);
strcpy(sinfo->prodname, prod.info.ident);
sinfo->seqno = prod.info.seqno;
sinfo->prodsz = prod.info.sz;
sinfo->next = NULL;
stat_size += strlen(sinfo->prodname);
if(shead == NULL)
{
shead = sinfo;
slast = sinfo;
}
else
{
slast->next = sinfo;
slast = sinfo;
}
}
/*}*/
griboff += griblen;
prod.info.seqno++;
break;
case -1:
GRIBDONE = 1;
break;
case -2:
log_error_q ("truncated grib file at: %d", prod.info.seqno);
GRIBDONE = 1;
break;
case -7:
log_error_q ("End sequence 7777 not found where expected: %d",
prod.info.seqno);
griboff += griblen;
log_error_q("resume looking at %d\0",griboff);
break;
default:
log_error_q ("unknown error %d\0", status);
griboff += griblen;
if (griboff >= statb.st_size)
GRIBDONE = 1;
break;
}
if (griboff >= statb.st_size)
GRIBDONE = 1;
}
log_notice_q ("munmap\0");
(void) munmap ((void *)prodmmap, statb.st_size);
if ( stat_size != 0 )
/*
* Add a status message to product queue
*/
{
char *statusmess;
log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt);
statusmess = calloc((30 * sinfo_cnt) + stat_size +
strlen(filename) + 128, sizeof(char));
if(statusmess == NULL)
{
log_syserr("could not malloc status message %ld\0",
stat_size);
}
else
{
char tmpprod[512];
sinfo = shead; slast = NULL;
status = set_timestamp(&prod.info.arrival);
/* ctime ends with \n\0" */
sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n",
filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec),
insert_sum,(long)statb.st_size);
while(sinfo != NULL)
{
sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus,
sinfo->seqno,sinfo->prodsz,sinfo->prodname);
strcat(statusmess,tmpprod);
slast = sinfo;
sinfo = sinfo->next;
free(slast->prodname);
free(slast);
}
shead = NULL;
sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno);
prod.info.ident = tmpprod;
prod.data = statusmess;
prod.info.sz = strlen(statusmess);
status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature);
status = set_timestamp(&prod.info.arrival);
status = pq_insert(pq, &prod);
if(log_is_enabled_info)
log_info_q("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug)) ;
free(statusmess);
prod.info.seqno++;
}
}
}
(void) close (fd);
}
}
exit(0);
}
