/** * Opens a file descriptor on `/dev/null` if it's closed. If the file * descriptor is open, then nothing happens. * * @param[in] fileno File descriptor * @param[in] flags `open()` flags * @retval 0 Success * @retval -1 Failure. log_add() called. */ int open_on_dev_null_if_closed( const int fileno, const int flags) { int status; if (fcntl(fileno, F_GETFD) >= 0) { status = 0; } else { status = -1; int fd = open("/dev/null", flags); if (fd < 0) { log_add_syserr("Couldn't open /dev/null: flags=%#X", flags); } else if (fd == fileno) { status = 0; } else { if (dup2(fd, fileno) < 0) { log_add_syserr("dup2() failure: fd=%d, fileno=%d", fd, fileno); } else { status = 0; } (void)close(fd); } } return status; }
static inline void lock( ProdIndexQueue* const fiq) { int status = pthread_mutex_lock(&fiq->mutex); if (status) log_add_syserr("Couldn't lock mutex"); }
/** * Ensures that a file descriptor will close if and when a function of the * `exec()` family is called. * * @param[in] fd File descriptor * @retval 0 Success * @retval -1 `fd` is not a valid open file descriptor. log_add() called. */ int ensure_close_on_exec( const int fd) { int status = fcntl(fd, F_GETFD); if (status == -1) { log_add_syserr("Couldn't get file descriptor flags: fd=%d", fd); } else if (status & FD_CLOEXEC) { status = 0; } else { status = fcntl(fd, F_SETFD, status | FD_CLOEXEC); if (status == -1) { log_add_syserr("Couldn't set file descriptor to close-on-exec: " "fd=%d", fd); } else { status = 0; } } return status; }
/** * There is data available on the feed. Read it into the buffer * then deal with what we got. * * @param ifd [in] File-descriptor of the input data-feed * @retval 0 Success * @retval ENOMEM Out of memory * @retval ENODATA End of input data-feed */ int feedTheXbuf(const int ifd) { int status; size_t nn = 0; /* space available in buffer */ ptrdiff_t remaining = (ptrdiff_t)theBuf->bufsiz - (theBuf->get - theBuf->base); if (remaining <= CHUNKSIZE) { if (theBuf->bufsiz >= maxProductSize) { log_warning_q( "Data-product would exceed %lu bytes. Resetting input buffer.", maxProductSize); justify_xbuf(theBuf, 0); } log_info_q("Expanding input buffer size to %lu\n", (unsigned long)(2 * theBuf->bufsiz)); theBuf = expand_xbuf(theBuf, theBuf->bufsiz); if (theBuf == NULL) { status = errno == 0 ? ENOMEM : errno; log_add_syserr("expand_xbuf"); log_flush_error(); return status; } } status = (*read_feed)(ifd, (char *)theBuf->put, CHUNKSIZE, &nn); if(status != ENOERR) { log_add_errno(status, "read_feed"); log_flush_error(); return status; } /* else */ if(nn == 0) return ENODATA; /* end of file */ /* else */ /* usual case */ /* assert(nn > 0); */ theBuf->cnt += nn; theBuf->put += nn; return ENOERR; }
/** * Sets an LDM product-identifier from a GRIB message. * * Not Atomic, * Idempotent, * Thread-safe * * @param[out] ident Product-identifier buffer. * @param[in] identSize Size of product-identifier buffer in bytes. * @param[in] decoded Decoded GRIB message. * @param[in] wmoHead WMO header associated with the GRIB message. * @retval 0 Success. \c *ident is set. * @retval 1 GRIB message has no data. * @retval 3 System error. */ static int setIdent( char* const ident, const size_t identSize, DecodedGrib2Msg* const decoded, const char* const wmoHead) { const size_t numFields = g2d_getNumFields(decoded); int status; if (0 == numFields) { log_add("GRIB message has no fields"); status = 1; } else { StringBuf* const prods = strBuf_new(127); /* initially empty */ if (NULL == prods) { log_add_syserr("Couldn't allocate string-buffer for products"); log_flush_error(); status = 3; } else { Gribmsg gribMsg; Geminfo gemInfo; int modelId; status = appendParameterNames(prods, decoded, &gribMsg, &gemInfo); if (0 == status) { status = getModelId(decoded, &modelId); if (0 == status) composeIdent(ident, identSize, decoded, &gribMsg, &gemInfo, prods, wmoHead, modelId); } strBuf_free(prods); } /* "prods" allocated */ } /* numFields > 0 */ return status; }
/** * Initializes this module. * * @param readfunct [in] The function that reads the data * @param maxProdSize [in] The size, in bytes, of the largest expected * data-product * @retval 0 Success * @retval ENOMEM Out of memory */ int initTheXbuf( int (*readfunct)(int ifd, char *buf, size_t nbytes, size_t *ngotp), const unsigned long maxProdSize) { read_feed = readfunct; maxProductSize = maxProdSize > INIT_CIRCBUFSIZE ? maxProdSize : INIT_CIRCBUFSIZE; if(theBuf == NULL) { theBuf = new_xbuf(INIT_CIRCBUFSIZE); if(theBuf == NULL) { const int status = errno == 0 ? ENOMEM : errno; log_add_syserr("new_xbuf"); log_flush_error(); return status; } } return ENOERR; }
/** * Returns the maximum number of open file descriptors. * * @retval -1 Failure. log_add() called. * @return The maximum number of open file descriptors */ static int open_max(void) { static int open_max = #ifdef OPEN_MAX OPEN_MAX; #else 0; if (open_max == 0) { errno = 0; open_max = sysconf(_SC_OPEN_MAX); if (open_max < 0) { if (errno == 0) { open_max = 256; // Indeterminate. 256 might be inadequate } else { log_add_syserr("Couldn't get maximum number of file " "descriptors"); } } } #endif return open_max; }
int main (int ac, char *av[]) { char *progname = av[0]; int status; int seq_start = 0; stat_info *sinfo, *shead = NULL, *slast = NULL; int statusoff=0; /* * Set up error logging */ if (log_init(progname)) { log_syserr("Couldn't initialize logging module"); exit(1); } const char* pqfname = getQueuePath(); /* * Check the environment for some options. * May be overridden by command line switches below. */ { const char *ldmpqfname = getenv ("LDMPQFNAME"); if (ldmpqfname != NULL) pqfname = ldmpqfname; } { extern int optind; extern int opterr; extern char *optarg; int ch; opterr = 1; while ((ch = getopt (ac, av, "vxl:q:f:s:S")) != EOF) switch (ch) { case 'v': if (!log_is_enabled_info) (void)log_set_level(LOG_LEVEL_INFO); break; case 'x': (void)log_set_level(LOG_LEVEL_DEBUG); break; case 'l': if (log_set_destination(optarg)) { log_syserr("Couldn't set logging destination to \"%s\"", optarg); exit(1); } break; case 'q': pqfname = optarg; break; case 's': seq_start = atoi (optarg); break; case 'f': feedtype = atofeedtypet (optarg); if (feedtype == NONE) { fprintf (stderr, "Unknown feedtype \"%s\"\n", optarg); usage (progname); } break; case 'S': statusoff=1; break; case '?': usage (progname); break; } setQueuePath(pqfname); ac -= optind; av += optind; if (ac < 1) usage (progname); } /* * register exit handler */ if (atexit (cleanup) != 0) { log_syserr ("atexit"); exit (1); } /* * set up signal handlers */ set_sigactions (); /* * who am i, anyway */ (void) strcpy (myname, ghostname ()); /* * open the product queue */ if ((status = pq_open (pqfname, PQ_DEFAULT, &pq))) { if (status > 0) { log_add_syserr("\"%s\" failed", pqfname); log_flush_error(); } else { log_error_q("\"%s\" failed: %s", pqfname, "Internal error"); } exit (2); } { char *filename; int fd; struct stat statb; product prod; unsigned char *prodmmap; MD5_CTX *md5ctxp = NULL; int gversion; /* * Allocate an MD5 context */ md5ctxp = new_MD5_CTX (); if (md5ctxp == NULL) { log_syserr ("new_md5_CTX failed"); exit (6); } /* These members are constant over the loop. */ prod.info.origin = myname; prod.info.feedtype = feedtype; prod.info.seqno = seq_start; /* * Open the file to be inserted and process */ while (ac > 0) { long insert_sum = 0; long sinfo_cnt = 0; long stat_size = 0; filename = *av; av++; ac--; log_notice_q ("open and memorymap %s\0", filename); fd = open (filename, O_RDONLY, 0); if (fd == -1) { log_syserr ("open: %s", filename); continue; } if (fstat (fd, &statb) == -1) { log_syserr ("fstat: %s", filename); (void) close (fd); continue; } if ((prodmmap = (unsigned char *) mmap (0, statb.st_size, PROT_READ, MAP_PRIVATE, fd, 0)) == MAP_FAILED) { log_syserr ("allocation failed"); } else { int GRIBDONE = 0; off_t griboff = 0; size_t griblen = 0; log_notice_q ("%ld bytes memory mapped\0", (long) statb.st_size); while (!GRIBDONE) { log_debug("griboff %d\0", (int) griboff); /* get offset of next grib product */ status = get_grib_info (prodmmap, statb.st_size, &griboff, &griblen, &gversion); switch (status) { case 0: prod.data = prodmmap + griboff; prod.info.sz = griblen; /* * revised MD5 calculation...using filename * to allow duplicate products in different files. */ MD5Init (md5ctxp); MD5Update (md5ctxp, (void *)filename, strlen(filename)); /*MD5Update (md5ctxp, (void *)prod.data, prod.info.sz);*/ if ( prod.info.sz > 10000 ) MD5Update (md5ctxp, (void *)prod.data, 10000); else MD5Update (md5ctxp, (void *)prod.data, prod.info.sz); MD5Final (prod.info.signature, md5ctxp); /*if (mm_md5 (md5ctxp, prod.data, prod.info.sz, prod.info.signature) != 0) { log_error_q ("could not compute MD5\0"); } else { */ prod.info.ident = (char *) malloc (KEYSIZE + 1); get_gribname (gversion, prod.data, prod.info.sz, filename, prod.info.seqno, prod.info.ident); /* * Do the deed */ status = set_timestamp (&prod.info.arrival); if (status != ENOERR) { log_add_syserr("could not set timestamp"); log_flush_error(); } /* * Insert the product */ status = pq_insert (pq, &prod); log_info_q ("%d %s\0", status, prod.info.ident); if ( status == ENOERR ) insert_sum += prod.info.sz; if (! statusoff ) { /* * Log this status */ sinfo_cnt++; sinfo = (stat_info *)malloc(sizeof(stat_info)); sinfo->insertstatus = status; sinfo->prodname = (char *)malloc(strlen(prod.info.ident)+1); strcpy(sinfo->prodname, prod.info.ident); sinfo->seqno = prod.info.seqno; sinfo->prodsz = prod.info.sz; sinfo->next = NULL; stat_size += strlen(sinfo->prodname); if(shead == NULL) { shead = sinfo; slast = sinfo; } else { slast->next = sinfo; slast = sinfo; } } /*}*/ griboff += griblen; prod.info.seqno++; break; case -1: GRIBDONE = 1; break; case -2: log_error_q ("truncated grib file at: %d", prod.info.seqno); GRIBDONE = 1; break; case -7: log_error_q ("End sequence 7777 not found where expected: %d", prod.info.seqno); griboff += griblen; log_error_q("resume looking at %d\0",griboff); break; default: log_error_q ("unknown error %d\0", status); griboff += griblen; if (griboff >= statb.st_size) GRIBDONE = 1; break; } if (griboff >= statb.st_size) GRIBDONE = 1; } log_notice_q ("munmap\0"); (void) munmap ((void *)prodmmap, statb.st_size); if ( stat_size != 0 ) /* * Add a status message to product queue */ { char *statusmess; log_notice_q("stats_size %ld %ld\0",stat_size,sinfo_cnt); statusmess = calloc((30 * sinfo_cnt) + stat_size + strlen(filename) + 128, sizeof(char)); if(statusmess == NULL) { log_syserr("could not malloc status message %ld\0", stat_size); } else { char tmpprod[512]; sinfo = shead; slast = NULL; status = set_timestamp(&prod.info.arrival); /* ctime ends with \n\0" */ sprintf(statusmess,"%s complete (%ld bytes) at %sInserted %ld of %ld\n", filename,(long)statb.st_size, ctime(&prod.info.arrival.tv_sec), insert_sum,(long)statb.st_size); while(sinfo != NULL) { sprintf(tmpprod,"%3d %5d %8d %s\n",sinfo->insertstatus, sinfo->seqno,sinfo->prodsz,sinfo->prodname); strcat(statusmess,tmpprod); slast = sinfo; sinfo = sinfo->next; free(slast->prodname); free(slast); } shead = NULL; sprintf(tmpprod,".status.%s %06d",filename, prod.info.seqno); prod.info.ident = tmpprod; prod.data = statusmess; prod.info.sz = strlen(statusmess); status = mm_md5(md5ctxp, prod.data, prod.info.sz, prod.info.signature); status = set_timestamp(&prod.info.arrival); status = pq_insert(pq, &prod); if(log_is_enabled_info) log_info_q("%s", s_prod_info(NULL, 0, &prod.info, log_is_enabled_debug)) ; free(statusmess); prod.info.seqno++; } } } (void) close (fd); } } exit(0); }