/**
* Opens a file descriptor on `/dev/null` if it's closed. If the file
* descriptor is open, then nothing happens.
*
* @param[in] fileno File descriptor
* @param[in] flags `open()` flags
* @retval 0 Success
* @retval -1 Failure. log_add() called.
*/
int open_on_dev_null_if_closed(
const int fileno,
const int flags)
{
int status;
if (fcntl(fileno, F_GETFD) >= 0) {
status = 0;
}
else {
status = -1;
int fd = open("/dev/null", flags);
if (fd < 0) {
log_add_syserr("Couldn't open /dev/null: flags=%#X", flags);
}
else if (fd == fileno) {
status = 0;
}
else {
if (dup2(fd, fileno) < 0) {
log_add_syserr("dup2() failure: fd=%d, fileno=%d", fd, fileno);
}
else {
status = 0;
}
(void)close(fd);
}
}
return status;
}
static inline void
lock(
ProdIndexQueue* const fiq)
{
int status = pthread_mutex_lock(&fiq->mutex);
if (status)
log_add_syserr("Couldn't lock mutex");
}
/**
* Ensures that a file descriptor will close if and when a function of the
* `exec()` family is called.
*
* @param[in] fd File descriptor
* @retval 0 Success
* @retval -1 `fd` is not a valid open file descriptor. log_add() called.
*/
int ensure_close_on_exec(
const int fd)
{
int status = fcntl(fd, F_GETFD);
if (status == -1) {
log_add_syserr("Couldn't get file descriptor flags: fd=%d", fd);
}
else if (status & FD_CLOEXEC) {
status = 0;
}
else {
status = fcntl(fd, F_SETFD, status | FD_CLOEXEC);
if (status == -1) {
log_add_syserr("Couldn't set file descriptor to close-on-exec: "
"fd=%d", fd);
}
else {
status = 0;
}
}
return status;
}
/**
* There is data available on the feed. Read it into the buffer
* then deal with what we got.
*
* @param ifd [in] File-descriptor of the input data-feed
* @retval 0 Success
* @retval ENOMEM Out of memory
* @retval ENODATA End of input data-feed
*/
int
feedTheXbuf(const int ifd)
{
int status;
size_t nn = 0;
/* space available in buffer */
ptrdiff_t remaining = (ptrdiff_t)theBuf->bufsiz - (theBuf->get - theBuf->base);
if (remaining <= CHUNKSIZE) {
if (theBuf->bufsiz >= maxProductSize) {
log_warning_q(
"Data-product would exceed %lu bytes. Resetting input buffer.",
maxProductSize);
justify_xbuf(theBuf, 0);
}
log_info_q("Expanding input buffer size to %lu\n",
(unsigned long)(2 * theBuf->bufsiz));
theBuf = expand_xbuf(theBuf, theBuf->bufsiz);
if (theBuf == NULL) {
status = errno == 0 ? ENOMEM : errno;
log_add_syserr("expand_xbuf");
log_flush_error();
return status;
}
}
status = (*read_feed)(ifd, (char *)theBuf->put, CHUNKSIZE, &nn);
if(status != ENOERR)
{
log_add_errno(status, "read_feed");
log_flush_error();
return status;
}
/* else */
if(nn == 0)
return ENODATA; /* end of file */
/* else */
/* usual case */
/* assert(nn > 0); */
theBuf->cnt += nn;
theBuf->put += nn;
return ENOERR;
}
/**
* Sets an LDM product-identifier from a GRIB message.
*
* Not Atomic,
* Idempotent,
* Thread-safe
*
* @param[out] ident Product-identifier buffer.
* @param[in] identSize Size of product-identifier buffer in bytes.
* @param[in] decoded Decoded GRIB message.
* @param[in] wmoHead WMO header associated with the GRIB message.
* @retval 0 Success. \c *ident is set.
* @retval 1 GRIB message has no data.
* @retval 3 System error.
*/
static int setIdent(
char* const ident,
const size_t identSize,
DecodedGrib2Msg* const decoded,
const char* const wmoHead)
{
const size_t numFields = g2d_getNumFields(decoded);
int status;
if (0 == numFields) {
log_add("GRIB message has no fields");
status = 1;
}
else {
StringBuf* const prods = strBuf_new(127); /* initially empty */
if (NULL == prods) {
log_add_syserr("Couldn't allocate string-buffer for products");
log_flush_error();
status = 3;
}
else {
Gribmsg gribMsg;
Geminfo gemInfo;
int modelId;
status = appendParameterNames(prods, decoded, &gribMsg, &gemInfo);
if (0 == status) {
status = getModelId(decoded, &modelId);
if (0 == status)
composeIdent(ident, identSize, decoded, &gribMsg, &gemInfo,
prods, wmoHead, modelId);
}
strBuf_free(prods);
} /* "prods" allocated */
} /* numFields > 0 */
return status;
}
/**
* Initializes this module.
*
* @param readfunct [in] The function that reads the data
* @param maxProdSize [in] The size, in bytes, of the largest expected
* data-product
* @retval 0 Success
* @retval ENOMEM Out of memory
*/
int
initTheXbuf(
int (*readfunct)(int ifd, char *buf, size_t nbytes, size_t *ngotp),
const unsigned long maxProdSize)
{
read_feed = readfunct;
maxProductSize = maxProdSize > INIT_CIRCBUFSIZE ? maxProdSize : INIT_CIRCBUFSIZE;
if(theBuf == NULL)
{
theBuf = new_xbuf(INIT_CIRCBUFSIZE);
if(theBuf == NULL)
{
const int status = errno == 0 ? ENOMEM : errno;
log_add_syserr("new_xbuf");
log_flush_error();
return status;
}
}
return ENOERR;
}
/**
* Returns the maximum number of open file descriptors.
*
* @retval -1 Failure. log_add() called.
* @return The maximum number of open file descriptors
*/
static int
open_max(void)
{
static int open_max =
#ifdef OPEN_MAX
OPEN_MAX;
#else
0;
if (open_max == 0) {
errno = 0;
open_max = sysconf(_SC_OPEN_MAX);
if (open_max < 0) {
if (errno == 0) {
open_max = 256; // Indeterminate. 256 might be inadequate
}
else {
log_add_syserr("Couldn't get maximum number of file "
"descriptors");
}
}
}
#endif
return open_max;
}
int
main (int ac, char *av[])
{
char *progname = av[0];
int status;
int seq_start = 0;
stat_info *sinfo, *shead = NULL, *slast = NULL;
int statusoff=0;
/*
* Set up error logging
*/
if (log_init(progname)) {
log_syserr("Couldn't initialize logging module");
exit(1);
}
const char* pqfname = getQueuePath();
/*
* Check the environment for some options.
* May be overridden by command line switches below.
*/
{
const char *ldmpqfname = getenv ("LDMPQFNAME");
if (ldmpqfname != NULL)
pqfname = ldmpqfname;
}
{
extern int optind;
extern int opterr;
extern char *optarg;
int ch;
opterr = 1;
while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF)
switch (ch)
{
case 'v':
if (!log_is_enabled_info)
(void)log_set_level(LOG_LEVEL_INFO);
break;
case 'x':
(void)log_set_level(LOG_LEVEL_DEBUG);
break;
case 'l':
if (log_set_destination(optarg)) {
log_syserr("Couldn't set logging destination to \"%s\"",
optarg);
exit(1);
}
break;
case 'q':
pqfname = optarg;
break;
case 's':
seq_start = atoi (optarg);
break;
case 'f':
feedtype = atofeedtypet (optarg);
if (feedtype == NONE)
{
fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg);
usage (progname);
}
break;
case 'S':
statusoff=1;
break;
case '?':
usage (progname);
break;
}
setQueuePath(pqfname);
ac -= optind;
av += optind;
if (ac < 1)
usage (progname);
}
/*
* register exit handler
*/
if (atexit (cleanup) != 0)
{
log_syserr ("atexit");
exit (1);
}
/*
* set up signal handlers
*/
set_sigactions ();
/*
* who am i, anyway
*/
(void) strcpy (myname, ghostname ());
/*
* open the product queue
*/
if ((status = pq_open (pqfname, PQ_DEFAULT, &pq)))
{
if (status > 0) {
log_add_syserr("\"%s\" failed", pqfname);
log_flush_error();
}
else {
log_error_q("\"%s\" failed: %s", pqfname, "Internal error");
}
exit (2);
}
{
char *filename;
int fd;
struct stat statb;
product prod;
unsigned char *prodmmap;
MD5_CTX *md5ctxp = NULL;
int gversion;
/*
* Allocate an MD5 context
*/
md5ctxp = new_MD5_CTX ();
if (md5ctxp == NULL)
{
log_syserr ("new_md5_CTX failed");
exit (6);
}
/* These members are constant over the loop. */
prod.info.origin = myname;
prod.info.feedtype = feedtype;
prod.info.seqno = seq_start;
/*
* Open the file to be inserted and process
*/
while (ac > 0)
{
long insert_sum = 0;
long sinfo_cnt = 0;
long stat_size = 0;
filename = *av;
av++;
ac--;
log_notice_q ("open and memorymap %s\0", filename);
fd = open (filename, O_RDONLY, 0);
if (fd == -1)
{
log_syserr ("open: %s", filename);
continue;
}
if (fstat (fd, &statb) == -1)
{
log_syserr ("fstat: %s", filename);
(void) close (fd);
continue;
}
if ((prodmmap = (unsigned char *) mmap (0, statb.st_size,
PROT_READ, MAP_PRIVATE, fd,
0)) == MAP_FAILED)
{
log_syserr ("allocation failed");
}
else
{
int GRIBDONE = 0;
off_t griboff = 0;
size_t griblen = 0;
log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size);
while (!GRIBDONE)
{
log_debug("griboff %d\0", (int) griboff);
/* get offset of next grib product */
status =
get_grib_info (prodmmap, statb.st_size, &griboff, &griblen,
&gversion);
switch (status)
{
case 0:
prod.data = prodmmap + griboff;
prod.info.sz = griblen;
/*
* revised MD5 calculation...using filename
* to allow duplicate products in different files.
*/
MD5Init (md5ctxp);
MD5Update (md5ctxp, (void *)filename, strlen(filename));
/*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/
if ( prod.info.sz > 10000 )
MD5Update (md5ctxp, (void *)prod.data, 10000);
else
MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);
MD5Final (prod.info.signature, md5ctxp);
/*if (mm_md5 (md5ctxp, prod.data, prod.info.sz,
prod.info.signature) != 0)
{
log_error_q ("could not compute MD5\0");
}
else
{ */
prod.info.ident = (char *) malloc (KEYSIZE + 1);
get_gribname (gversion, prod.data, prod.info.sz,
filename, prod.info.seqno,
prod.info.ident);
/*
* Do the deed
*/
status = set_timestamp (&prod.info.arrival);
if (status != ENOERR)
{
log_add_syserr("could not set timestamp");
log_flush_error();
}
/*
* Insert the product
*/
status = pq_insert (pq, &prod);
log_info_q ("%d %s\0", status, prod.info.ident);
if ( status == ENOERR )
insert_sum += prod.info.sz;
if (! statusoff )
{
/*
* Log this status
*/
sinfo_cnt++;
sinfo = (stat_info *)malloc(sizeof(stat_info));
sinfo->insertstatus = status;
sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1);
strcpy(sinfo->prodname, prod.info.ident);
sinfo->seqno = prod.info.seqno;
sinfo->prodsz = prod.info.sz;
sinfo->next = NULL;
stat_size += strlen(sinfo->prodname);
if(shead == NULL)
{
shead = sinfo;
slast = sinfo;
}
else
{
slast->next = sinfo;
slast = sinfo;
}
}
/*}*/
griboff += griblen;
prod.info.seqno++;
break;
case -1:
GRIBDONE = 1;
break;
case -2:
log_error_q ("truncated grib file at: %d", prod.info.seqno);
GRIBDONE = 1;
break;
case -7:
log_error_q ("End sequence 7777 not found where expected: %d",
prod.info.seqno);
griboff += griblen;
log_error_q("resume looking at %d\0",griboff);
break;
default:
log_error_q ("unknown error %d\0", status);
griboff += griblen;
if (griboff >= statb.st_size)
GRIBDONE = 1;
break;
}
if (griboff >= statb.st_size)
GRIBDONE = 1;
}
log_notice_q ("munmap\0");
(void) munmap ((void *)prodmmap, statb.st_size);
if ( stat_size != 0 )
/*
* Add a status message to product queue
*/
{
char *statusmess;
log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt);
statusmess = calloc((30 * sinfo_cnt) + stat_size +
strlen(filename) + 128, sizeof(char));
if(statusmess == NULL)
{
log_syserr("could not malloc status message %ld\0",
stat_size);
}
else
{
char tmpprod[512];
sinfo = shead; slast = NULL;
status = set_timestamp(&prod.info.arrival);
/* ctime ends with \n\0" */
sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n",
filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec),
insert_sum,(long)statb.st_size);
while(sinfo != NULL)
{
sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus,
sinfo->seqno,sinfo->prodsz,sinfo->prodname);
strcat(statusmess,tmpprod);
slast = sinfo;
sinfo = sinfo->next;
free(slast->prodname);
free(slast);
}
shead = NULL;
sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno);
prod.info.ident = tmpprod;
prod.data = statusmess;
prod.info.sz = strlen(statusmess);
status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature);
status = set_timestamp(&prod.info.arrival);
status = pq_insert(pq, &prod);
if(log_is_enabled_info)
log_info_q("%s", s_prod_info(NULL, 0, &prod.info,
log_is_enabled_debug)) ;
free(statusmess);
prod.info.seqno++;
}
}
}
(void) close (fd);
}
}
exit(0);
}
