Пример #1
0
/**
 * Executes a product-maker.
 *
 * This function is thread-compatible but not thread-safe.
 *
 * @retval (void*)0    The FIFO was closed.
 * @retval (void*)1    Usage failure. \c log_start() called.
 * @retval (void*)2    O/S failure. \c log_start() called.
 * @retval (void*)-1   Retransmission failure. \c log_start() called.
 */
void* pmStart(
    void* const         arg)          /**< [in/out] Pointer to the
                                        *  product-maker to be executed */
{
    ProductMaker* const productMaker = (ProductMaker*)arg;
    int                 status;
    Fifo* const         fifo = productMaker->fifo;
    unsigned char*      buf = productMaker->buf;
    sbn_struct*         sbn = &productMaker->sbn;
    pdh_struct*         pdh = &productMaker->pdh;
    psh_struct*         psh = &productMaker->psh;
    pdb_struct*         pdb = &productMaker->pdb;
    ccb_struct*         ccb = &productMaker->ccb;
    unsigned long       last_sbn_seqno;
    unsigned long       last_sbn_runno = ULONG_MAX;
    int                 PNGINIT = 0;
    char*               memheap = NULL;
    MD5_CTX*            md5ctxp = productMaker->md5ctxp;
    int                 logResync = 1;
    prodstore           prod;

#ifdef RETRANS_SUPPORT
    long cpio_addr_tmp;
    int cpio_fd_tmp;
 /*   char transfer_type[10]={0};*/
    int retrans_val,idx;
    long retrans_tbl_size;
    time_t orig_arrive_time;
    int new_key; /* shm key */
    ACQ_TABLE *acq_tbl = NULL;
    long num_prod_discards=0;
    int save_prod = 1;
    int discard_prod = 0;
    long proc_orig_prod_seqno_last_save=0;
#endif


    prod.head = NULL;
    prod.tail = NULL;

	/*** For Retranmission Purpose  ***/
#ifdef RETRANS_SUPPORT
	if (ulogIsDebug())
         udebug(" retrans_xmit_enable = %d   transfer_type = %s sbn_channel_name=%s \n",retrans_xmit_enable,transfer_type,sbn_channel_name);

       if((retrans_xmit_enable == OPTION_ENABLE) && (!strcmp(transfer_type,"MHS") || !strcmp(transfer_type,"mhs"))){
                idx = get_cpio_addr(mcastAddr);
                if( idx >= 0 && idx < NUM_CPIO_ENTRIES){
                    global_cpio_fd = cpio_tbl[idx].cpio_fd;
                    global_cpio_addr = cpio_tbl[idx].cpio_addr;
	            if (ulogIsDebug())
 	              udebug("Global cpio_addr  = 0x%x Global cpio_fd = %d \n",global_cpio_addr,global_cpio_fd);
                }else{
                    uerror("Invalid multicast address provided");
	  	    return (void*)-1;
                 }

		 retrans_tbl_size = sizeof(PROD_RETRANS_TABLE);

		/** Modified to setup retrans table on per channel basis - Sathya - 14-Mar'2013 **/

		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * GET_RETRANS_CHANNEL_ENTRIES(sbn_type));

                /****   
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC1);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC2);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC3);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_GOES_EAST);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NOAAPORT_OPT);
		****/


		p_prod_retrans_table = (PROD_RETRANS_TABLE *) malloc (retrans_tbl_size);
		if(p_prod_retrans_table == NULL){
		   uerror("Unable to allocate memory for retrans table..Quitting.\n");
	  	   return (void*)-1;
		}

		if( init_retrans(&p_prod_retrans_table) < 0 ){
		  uerror("Error in initializing retrans table \n");
		  if(p_prod_retrans_table)
			free(p_prod_retrans_table);
	  	  return (void*)-1;
		}	

       GET_SHMPTR(global_acq_tbl,ACQ_TABLE,ACQ_TABLE_SHMKEY,DEBUGGETSHM);

	if (ulogIsDebug())
	  udebug("Global acquisition table = 0x%x cpio_fd = %d \n",global_acq_tbl,global_cpio_fd);
	acq_tbl = &global_acq_tbl[global_cpio_fd];

	if (ulogIsDebug())
	    udebug("Obtained acquisition table = 0x%x \n",acq_tbl);

	/* ACQ_TABLE already initialized in acq_ldm_getshm */
	/*
	if(init_acq_table(acq_tbl) < 0){
		uerror("Unable to initialize acq table\n");
		exit(-1);
	}
	*/
	
	 buff_hdr = (BUFF_HDR *) malloc(sizeof(BUFF_HDR));
	 if(init_buff_hdr(buff_hdr) < 0){
		uerror("Unalbe to initialize buffer header \n");
		if(acq_tbl)
			free(acq_tbl);
		if(p_prod_retrans_table)
		  free(p_prod_retrans_table);
	  	return (void*)-1;
	 }

	 acq_tbl->pid = getpid();
	 acq_tbl->link_id = global_cpio_fd;
	 acq_tbl->link_addr = global_cpio_addr;
	 if(ulogIsVerbose()){
 	  uinfo("Initialized acq_tbl  = 0x%x & buff_hdr = 0x%x pid = %d \n",acq_tbl,buff_hdr,acq_tbl->pid);
	  uinfo("Global link id = %d  Global link addr = %ld \n",acq_tbl->link_id,acq_tbl->link_addr);
	  uinfo("acq_tbl->read_distrib_enable = 0x%x \n",acq_tbl->read_distrib_enable);
	 }
    }
	   /*** For Retranmission Purpose  ***/
#endif

    for (;;) {
        unsigned char       b1;
        long                IOFF;
        int                 NWSTG;
        int                 GOES;
        int                 PROD_COMPRESSED;
        size_t              heapcount;
        size_t              heapsize;
        char                PROD_NAME[1024];
        int                 dataoff;
        int                 datalen;
        datastore*          pfrag;
        int                 nscan;
        int                 deflen;
        static const char*  FOS_TRAILER = "\015\015\012\003";
        int                 cnt;

        /* Look for first byte == 255  and a valid SBN checksum */
        if ((status = fifoRead(fifo, buf, 1)) != 0) {
            if (3 == status)
                status = 0;
            break;
        }
        if ((b1 = (unsigned char)buf[0]) != 255) {
            if (logResync) {
                uinfo("Trying to resync %u", b1);
                logResync = 0;
            }
            continue;
        }
        logResync = 1;

        if (fifoRead(fifo, buf + 1, 15) != 0) {
            if (ulogIsDebug())
                udebug("couldn't read 16 bytes for sbn");
            continue;
        }

        while ((status = readsbn(buf, sbn)) != 0) {
            if (ulogIsDebug())
                udebug("Not SBN start");

            IOFF = 1;

            while ((IOFF < 16) && 
                    ((b1 = (unsigned char) buf[IOFF]) != 255))
                IOFF++;

            if (IOFF > 15) {
                break;
            }
            else {
                int ch;

                for (ch = IOFF; ch < 16; ch++)
                    buf[ch - IOFF] = buf[ch];

                if (fifoRead(fifo, buf + 16 - IOFF, IOFF)
                        != 0) {
                    if (ulogIsDebug())
                        udebug("Couldn't read bytes for SBN, resync");
                    break;
                }
            }
        }

        if (status != 0) {
            if (ulogIsDebug())
                udebug("SBN status continue");
            continue;
        }

        IOFF = 0;

        if (fifoRead(fifo, buf + 16, 16) != 0) {
            if (ulogIsDebug())
                udebug("error reading Product Definition Header");
            continue;
        }

#ifdef RETRANS_SUPPORT
		/* Update acq table stats - Begin */
		if(retrans_xmit_enable == OPTION_ENABLE){
			buff_hdr->read_channel_type = sbn->datastream;
		}
		/* Update acq table stats - End */
#endif

        if (ulogIsDebug())
            udebug("***********************************************");
        if (last_sbn_runno != sbn->runno) {
            last_sbn_runno = sbn->runno;
        }
        else {
            unsigned long   delta = sbn->seqno - last_sbn_seqno;
#           define          MAX_SEQNO 0xFFFFFFFFu

            if (0 == delta || MAX_SEQNO/2 < delta) {
                uwarn("Retrograde packet number: previous=%lu, latest=%lu, "
                        "difference=%lu", last_sbn_seqno, sbn->seqno, 
                        0 == delta ? 0ul : MAX_SEQNO - delta + 1);
            }
            else {
                if (1 != delta) {
                    unsigned long   gap = delta - 1;

                    uwarn("Gap in packet sequence: %lu to %lu [skipped %lu]",
                             last_sbn_seqno, sbn->seqno, gap);

                    (void)pthread_mutex_lock(&productMaker->mutex);
                    productMaker->nmissed += gap;
                    (void)pthread_mutex_unlock(&productMaker->mutex);
                }

                (void)pthread_mutex_lock(&productMaker->mutex);
                productMaker->npackets++;
                (void)pthread_mutex_unlock(&productMaker->mutex);
            }                           /* non-retrograde packet number */
        }                               /* "last_sbn_seqno" initialized */
        last_sbn_seqno = sbn->seqno;

        if (ulogIsVerbose())
            uinfo("SBN seqnumber %ld", sbn->seqno);
        if (ulogIsVerbose())
            uinfo("SBN datastream %d command %d", sbn->datastream,
                sbn->command);
        if (ulogIsDebug())
            udebug("SBN version %d length offset %d", sbn->version, sbn->len);
        if (((sbn->command != 3) && (sbn->command != 5)) || 
                (sbn->version != 1)) {
            uerror("Unknown sbn command/version %d PUNT", sbn->command);
            continue;
        }

        switch (sbn->datastream) {
        case 5:       /* nwstg */
        case 6:       /* nwtg2 */
        case 7:       /* polarsat */
        case 8:       /* NOAA Weather Wire Service (NWWS) */
        case 9:
        case 10:
        case 11:
        case 12:      /* GOES-R */
        case 13:      /* GOES-R */
            NWSTG = 1;
            GOES = 0;
            break;
        case 1:       /* GINI GOES */
        case 2:       /* GINI GOES */
        case 4:       /* OCONUS */
            NWSTG = 0;
            GOES = 1;
            break;
        default:
            uerror("Unknown NOAAport channel %d PUNT", sbn->datastream);
            continue;
        }

        /* End of SBN version low 4 bits */

        if (readpdh(buf + IOFF + sbn->len, pdh) == -1) {
            uerror("problem with pdh, PUNT");
            continue;
        }
        if (pdh->len > 16) {
            if (fifoRead(fifo, buf + sbn->len + 16,
                        pdh->len - 16) != 0)
                continue;
        }

        if (ulogIsDebug())
            udebug("Product definition header version %d pdhlen %d",
                pdh->version, pdh->len);

        if (pdh->version != 1) {
            uerror("Error: PDH transfer type %u, PUNT", pdh->transtype);
            continue;
        }
        else if (ulogIsDebug()) {
            udebug("PDH transfer type %u", pdh->transtype);
        }

        if ((pdh->transtype & 8) > 0)
            uerror("Product transfer flag error %u", pdh->transtype);
        if ((pdh->transtype & 32) > 0)
            uerror("Product transfer flag error %u", pdh->transtype);

        if ((pdh->transtype & 16) > 0) {
            PROD_COMPRESSED = 1;

            if (ulogIsDebug())
                udebug("Product transfer flag compressed %u", pdh->transtype);
        }
        else {
            PROD_COMPRESSED = 0;
        }

        if (ulogIsDebug())
            udebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen,
                pdh->pshlen);
        if (ulogIsDebug())
            udebug("blocks per record %ld records per block %ld\n",
                pdh->blocks_per_record, pdh->records_per_block);
        if (ulogIsDebug())
            udebug("product seqnumber %ld block number %ld data block size "
                "%ld", pdh->seqno, pdh->dbno, pdh->dbsize);

        /* Stop here if no psh */
        if ((pdh->pshlen == 0) && (pdh->transtype == 0)) {
            IOFF = IOFF + sbn->len + pdh->len;
            continue;
        }

#ifdef RETRANS_SUPPORT
		/** Update acquisition table statistics  **/
		if(retrans_xmit_enable == OPTION_ENABLE){
				acq_tbl->read_tot_buff_read++;
		}
#endif
        if (pdh->pshlen != 0) {
            if (fifoRead(fifo, buf + sbn->len + pdh->len,
                        pdh->pshlen) != 0) {
                uerror("problem reading psh");
                continue;
            }
            else {
                if (ulogIsDebug())
                    udebug("read psh %d", pdh->pshlen);
            }

            /* Timing block */
            if (sbn->command == 5) {
                if (ulogIsDebug())
                    udebug("Timing block recieved %ld %ld\0", psh->olen,
                        pdh->len);
                /*
                 * Don't step on our psh of a product struct of prod in
                 * progress.
                 */
                continue;
            }

            if (readpsh(buf + IOFF + sbn->len + pdh->len, psh) == -1) {
                uerror("problem with readpsh");
                continue;
            }
            if (psh->olen != pdh->pshlen) {
                uerror("ERROR in calculation of psh len %ld %ld", psh->olen,
                    pdh->len);
                continue;
            }
            if (ulogIsDebug())
                udebug("len %ld", psh->olen);
            if (ulogIsDebug())
                udebug("product header flag %d, version %d", psh->hflag,
                    psh->version);
            if (ulogIsDebug())
                udebug("prodspecific data length %ld", psh->psdl);
            if (ulogIsDebug())
                udebug("bytes per record %ld", psh->bytes_per_record);
            if (ulogIsDebug())
                udebug("Fragments = %ld category %d ptype %d code %d",
                    psh->frags, psh->pcat, psh->ptype, psh->pcode);
            if (psh->frags < 0)
                uerror("check psh->frags %d", psh->frags);
            if (psh->origrunid != 0)
                uerror("original runid %d", psh->origrunid);
            if (ulogIsDebug())
                udebug("next header offset %ld", psh->nhoff);
            if (ulogIsDebug())
                udebug("original seq number %ld", psh->seqno);
            if (ulogIsDebug())
                udebug("receive time %ld", psh->rectime);
            if (ulogIsDebug())
                udebug("transmit time %ld", psh->transtime);
            if (ulogIsDebug())
                udebug("run ID %ld", psh->runid);
            if (ulogIsDebug())
                udebug("original run id %ld", psh->origrunid);

#ifdef RETRANS_SUPPORT
				/* Update acq table stats - Begin */
			if(retrans_xmit_enable == OPTION_ENABLE){
			
				buff_hdr->buff_data_length = pdh->dbsize;
				if(pdh->dbno == 0) {
						/* Assume first block */
					acq_tbl->proc_base_prod_type_last = psh->ptype;
					acq_tbl->proc_base_prod_cat_last = psh->pcat;
					acq_tbl->proc_base_prod_code_last = psh->pcode;
					acq_tbl->proc_prod_NCF_rcv_time = (time_t)psh->rectime;
					acq_tbl->proc_prod_NCF_xmit_time = (time_t)psh->transtime;
					if(psh->hflag & XFR_PROD_RETRANSMIT){
					   acq_tbl->proc_orig_prod_seqno_last = psh->seqno;
					   acq_tbl->proc_orig_prod_run_id = psh->origrunid;
					   if(ulogIsDebug())
					     udebug("ORIG SEQ# = %ld	CURR SEQ#: %ld \n",acq_tbl->proc_orig_prod_seqno_last,pdh->seqno);
						}else{
						   acq_tbl->proc_orig_prod_seqno_last = 0;
						   acq_tbl->proc_orig_prod_run_id = 0;
						}
					acq_tbl->proc_prod_run_id = psh->runid;
					buff_hdr->buff_datahdr_length = psh->psdl;
					time(&acq_tbl->proc_prod_start_time);
					acq_tbl->proc_tot_prods_handled++;
				}else{
						buff_hdr->buff_datahdr_length = 0;
				 }
				buff_hdr->proc_prod_seqno= pdh->seqno;
				buff_hdr->proc_blkno = pdh->dbno;
				buff_hdr->proc_sub_code = 0;
				buff_hdr->proc_prod_flag = pdh->transtype;
					
				acq_tbl->proc_base_channel_type_last = buff_hdr->read_channel_type;
				buff_hdr->proc_prod_type = acq_tbl->proc_base_prod_type_last;
				buff_hdr->proc_prod_code = acq_tbl->proc_base_prod_code_last;
				buff_hdr->proc_prod_cat = acq_tbl->proc_base_prod_cat_last;
					
				acq_tbl->proc_prod_bytes_read = buff_hdr->buff_data_length;
						
				/* Check prod_seqno for lost products */
				if((buff_hdr->proc_prod_seqno - acq_tbl->proc_base_prod_seqno_last) != 1){
					do_prod_lost(buff_hdr,acq_tbl);
				}
				retrans_val = prod_retrans_ck(acq_tbl, buff_hdr, &orig_arrive_time);
				log_buff[0] = '\0'; 		
				if((retrans_val == PROD_DUPLICATE_DISCARD) ||
					((retrans_val == PROD_DUPLICATE_MATCH) &&
					(acq_tbl->proc_retransmit_ctl_flag & ENABLE_RETRANS_DUP_MATCH_DISCARD)) ||
					((retrans_val == PROD_DUPLICATE_NOMATCH) &&
					(acq_tbl->proc_retransmit_ctl_flag & ENABLE_RETRANS_DUP_NOMATCH_DISCARD))){
						/* Log product details and discard the product */
						strcpy(log_buff,"DISCARD");
						if(acq_tbl->proc_orig_prod_seqno_last != 0){
							strcat(log_buff, "/RETRANS");
						}
								
						log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last,
									buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno,
									buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,orig_arrive_time);
						save_prod = 0;
						acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
						/* Current prod discarded and continue with next */
						/*continue; */
					}else{
						if(retrans_val == PROD_DUPLICATE_NOMATCH){
							strcpy(log_buff,"SAVE RETRANS");
							log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last,
							buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno,
							buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,acq_tbl->proc_prod_start_time);
					  }
				   }
			  }
#endif

            if (prod.head != NULL) {
                uerror("OOPS, start of new product [%ld ] with unfinished "
                    "product %ld", pdh->seqno, prod.seqno);

#ifdef RETRANS_SUPPORT
				/* Request retrans when prod is partially received but before completion */
				/* if there is frame error and continue with different prod then, we need */
				/* to abort the old prod and clear retrans table. */
				if((retrans_xmit_enable == OPTION_ENABLE) /*&& (pdh->dbno != 0)*/){
				 acq_tbl->proc_acqtab_prodseq_errs++;
				 if(proc_orig_prod_seqno_last_save != acq_tbl->proc_orig_prod_seqno_last){
				 /* Clear retrans table for the orig prod if the previous prod is retrans */
				 /* of original prod  */
				   prod_retrans_abort_entry(acq_tbl, proc_orig_prod_seqno_last_save, RETRANS_RQST_CAUSE_RCV_ERR);
				 }
				 prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
				 /* Update Statistics */
				 acq_tbl->proc_tot_prods_lost_errs++;
				  /* For now, generate retrans request only for non-imagery products */
				 if(!((buff_hdr->proc_prod_cat == PROD_CAT_IMAGE) && 
					   (PROD_TYPE_NESDIS_HDR_TRUE(buff_hdr->proc_prod_type)))){
				       generate_retrans_rqst(acq_tbl,prod.seqno , prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
				 }
				   acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
				}
#endif
                ds_free();

                prod.head = NULL;
                prod.tail = NULL;

                if (PNGINIT != 0) {
                    pngout_end();
                    PNGINIT = 0;
                }

                uerror("Product definition header version %d pdhlen %d",
                        pdh->version, pdh->len);
                uerror("PDH transfer type %u", pdh->transtype);

                if ((pdh->transtype & 8) > 0)
                    uerror("Product transfer flag error %u", pdh->transtype);
                if ((pdh->transtype & 32) > 0)
                    uerror("Product transfer flag error %u", pdh->transtype);

                uerror("header length %ld [pshlen = %d]",
                    pdh->len + pdh->pshlen, pdh->pshlen);
                uerror("blocks per record %ld records per block %ld",
                    pdh->blocks_per_record, pdh->records_per_block);
                uerror("product seqnumber %ld block number %ld data block "
                    "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize);
                uerror("product header flag %d", psh->hflag);
                uerror("prodspecific data length %ld", psh->psdl);
                uerror("bytes per record %ld", psh->bytes_per_record);
                uerror("Fragments = %ld category %d", psh->frags, psh->pcat);

                if (psh->frags < 0)
                    uerror("check psh->frags %d", psh->frags);
                if (psh->origrunid != 0)
                    uerror("original runid %d", psh->origrunid);

                uerror("next header offset %ld", psh->nhoff);
                uerror("original seq number %ld", psh->seqno);
                uerror("receive time %ld", psh->rectime);
                uerror("transmit time %ld", psh->transtime);
                uerror("run ID %ld", psh->runid);
                uerror("original run id %ld", psh->origrunid);
            }

            prod.seqno = pdh->seqno;
            prod.nfrag = psh->frags;

            ds_init(prod.nfrag);

            /* NWSTG CCB = dataoff, WMO = dataoff + 24 */

            if (fifoRead(fifo, buf + sbn->len + pdh->len + 
                        pdh->pshlen, pdh->dbsize) != 0) {
                uerror("problem reading datablock");
                continue;
            }
            if (sbn->datastream == 4) {
                if (psh->pcat != 3) {
                    GOES = 0;
                    NWSTG = 1;
                }
            }

            heapcount = 0;

            MD5Init(md5ctxp);

            if (GOES == 1) {
                if (readpdb(buf + IOFF + sbn->len + pdh->len + 
                            pdh->pshlen,
                        psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) {
                    uerror("Error reading pdb, punt");
                    continue;
                }

                (void)memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                if (ulogIsDebug())
                    udebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len,
                        pdh->pshlen, sbn->len + pdh->len + pdh->pshlen,
                        pdb->len);

                /* Data starts at first block after pdb */
                ccb->len = 0;
                heapsize = prodalloc(psh->frags, 5152, &memheap);
            }
            if (NWSTG == 1) {
                memset(psh->pname, 0, sizeof(psh->pname));

                if (readccb(buf + IOFF + sbn->len + pdh->len + 
                            pdh->pshlen,
                        ccb, psh, pdh->dbsize) == -1)
                    uerror("Error reading ccb, using default name");
                if (ulogIsDebug())
                    udebug("look at ccb start %d %d", ccb->b1, ccb->len);

                if (ulogIsVerbose())
                    uinfo("%s", psh->pname);

                memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                heapsize = prodalloc(psh->frags, 4000 + 15, &memheap);
                /*
                 * We will only compute md5 checksum on the data, 11 FOS
                 * characters at start
                 */
                sprintf(memheap, "\001\015\015\012%03d\040\015\015\012",
                    ((int) pdh->seqno) % 1000);

                heapcount += 11;

                if (psh->metaoff > 0)
                    psh->metaoff = psh->metaoff + 11;
            }
        }
        else {
            /* If a continuation record...don't let psh->pcat get missed */
            if ((sbn->datastream == 4) && (psh->pcat != 3)) {
                GOES = 0;
                NWSTG = 1;
            }

            ccb->len = 0;

            if (ulogIsDebug())
                udebug("continuation record");

#ifdef RETRANS_SUPPORT
			if(retrans_xmit_enable == OPTION_ENABLE){
					 buff_hdr->buff_data_length = pdh->dbsize;
					 buff_hdr->buff_datahdr_length = 0;
					 buff_hdr->proc_prod_seqno= pdh->seqno;
					 buff_hdr->proc_blkno = pdh->dbno;
					 buff_hdr->proc_sub_code = 0;
					 buff_hdr->proc_prod_flag = pdh->transtype;
					 
					 acq_tbl->proc_base_channel_type_last = buff_hdr->read_channel_type;
					 buff_hdr->proc_prod_type = acq_tbl->proc_base_prod_type_last;
					 buff_hdr->proc_prod_code = acq_tbl->proc_base_prod_code_last;
					 buff_hdr->proc_prod_cat = acq_tbl->proc_base_prod_cat_last;
				 
					 acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;
					 
			  }
#endif

            if ((pdh->transtype & 4) > 0) {
                psh->frags = 0;
            }
            if (fifoRead(fifo, buf + sbn->len + pdh->len + 
                        pdh->pshlen, pdh->dbsize) != 0) {
                uerror("problem reading datablock (cont)");
                continue;
            }
            if (prod.head == NULL) {
                if (ulogIsVerbose())
                    uinfo("found data block before header, "
                        "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno);
                continue;
            }
        }

        /* Get the data */
        dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len;
        datalen = pdh->dbsize - ccb->len;

        if (ulogIsDebug())
            udebug("look at datalen %d", datalen);

        pfrag = ds_alloc();
        pfrag->seqno = pdh->seqno;
        pfrag->fragnum = pdh->dbno;
        pfrag->recsiz = datalen;
        pfrag->offset = heapcount;
        pfrag->next = NULL;

        if (GOES == 1) {
            if (pfrag->fragnum > 0) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) || 
                        (pfrag->seqno != prod.seqno)) {
                    uerror("Missing GOES fragment in sequence, "
                        "last %d/%d this %d/%d\0", prod.tail->fragnum,
                        prod.seqno, pfrag->fragnum, pfrag->seqno);

#ifdef RETRANS_SUPPORT
					if(retrans_xmit_enable == OPTION_ENABLE){
					  acq_tbl->proc_acqtab_prodseq_errs++;
					  do_prod_mismatch(acq_tbl,buff_hdr);
					  acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
					}
#endif

					ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }

                if ((PNGINIT != 1) && (!PROD_COMPRESSED)) {
                    uerror("failed pnginit %d %d %s", sbn->datastream,
                            psh->pcat, PROD_NAME);
                    continue;
                }
                if (pdh->records_per_block < 1) {
                    uerror("records_per_block %d blocks_per_record %d "
                        "nx %d ny %d", pdh->records_per_block,
                        pdh->blocks_per_record, pdb->nx, pdb->ny);
                    uerror("source %d sector %d channel %d", pdb->source,
                        pdb->sector, pdb->channel);
                    uerror("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                        "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year,
                        pdb->month, pdb->day, pdb->hour, pdb->minute,
                        pdb->second, pdb->sechunds);
                    uerror("pshname %s", psh->pname);
                }
                if (!PROD_COMPRESSED) {
                    for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) {
                        if (ulogIsDebug())
                            udebug("png write nscan %d", nscan);
                        if (nscan >= pdh->records_per_block) {
                            uerror("nscan exceeding records per block %d [%d "
                                "%d %d]", pdh->records_per_block, nscan,
                                pdb->nx, pdh->dbsize);
                        }
                        else {
                          pngwrite(buf + dataoff + (nscan * pdb->nx));
                        }
                    }
                }
                else {
                    memcpy(memheap + heapcount, buf + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
            }
            else {
                if (!PROD_COMPRESSED) {
                    png_set_memheap(memheap, md5ctxp);
                    png_header(buf + dataoff, datalen);
                    /*
                     * Add 1 to number of scanlines, image ends with 
                     * f0f0f0f0...
                     */
                    pngout_init(pdb->nx, pdb->ny + 1);

                    PNGINIT = 1;
                }
                else {
                    memcpy(memheap + heapcount, buf + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
                unotice("records_per_block %d blocks_per_record %d nx %d ny %d",
                    pdh->records_per_block, pdh->blocks_per_record, pdb->nx,
                    pdb->ny);
                unotice("source %d sector %d channel %d", pdb->source,
                    pdb->sector, pdb->channel);
                unotice("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                    "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month,
                    pdb->day, pdb->hour, pdb->minute, pdb->second,
                    pdb->sechunds);
                unotice("pshname %s", psh->pname);
            }
            deflen = 0;
#ifdef RETRANS_SUPPORT
			if(retrans_xmit_enable == OPTION_ENABLE){
			   if(buff_hdr->proc_blkno != 0){
		          /*acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;*/
				  acq_tbl->proc_prod_bytes_read += datalen;
	           }
            }
#endif
       	}
        else {
            /* If the product already has a FOS trailer, don't add
             * another....this will match what pqing(SDI) sees
             */
            if ((prod.nfrag != 0) && (prod.tail != NULL)) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) ||
                        (pfrag->seqno != prod.seqno)) {
                    uerror("Missing fragment in sequence, last %d/%d this "
                        "%d/%d\0", prod.tail->fragnum, prod.seqno,
                        pfrag->fragnum, pfrag->seqno);

#ifdef RETRANS_SUPPORT
                                      if(retrans_xmit_enable == OPTION_ENABLE){
                                         acq_tbl->proc_acqtab_prodseq_errs++;
                                         if(ulogIsDebug())
                                            udebug("do_prod_mismatch() proc_base_prod_seqno_last = %d \n",
					                                    acq_tbl->proc_base_prod_seqno_last);
                                            do_prod_mismatch(acq_tbl,buff_hdr);
                                            acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
                                      }
#endif

                    ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }
            }
            if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
                char testme[4];

                while (datalen > 4) {
                    memcpy(testme, buf + (dataoff + datalen - 4), 4);

                    if (memcmp(testme, FOS_TRAILER, 4) == 0) {
                        datalen -= 4;

                        if (ulogIsDebug())
                            udebug("removing FOS trailer from %s", PROD_NAME);
                    }
                    else {
                        break;
                    }
                }
            }
            if (heapcount + datalen > heapsize) {
                /*
                 * this above wasn't big enough heapsize =
                 * prodalloc(psh->frags,4000+15,&memheap);
                 */
                uerror("Error in heapsize %d product size %d [%d %d], Punt!\0",
                    heapsize, (heapcount + datalen), heapcount, datalen);
				
#ifdef RETRANS_SUPPORT
				if(retrans_xmit_enable == OPTION_ENABLE){ 	
			      /* Update Statistics */
					acq_tbl->proc_tot_prods_lost_errs++;
					/*  Abort entry and request retransmission */
					prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
					generate_retrans_rqst(acq_tbl, prod.seqno, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
					if(acq_tbl->proc_orig_prod_seqno_last != 0){
						strcpy(log_buff, "RETRANS");
					}
					log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last,
								 buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno,
								 buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,
								 acq_tbl->proc_prod_start_time);

		      		acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
	      		}
#endif

                continue;
            }

            memcpy(memheap + heapcount, buf + dataoff, datalen);

            deflen = datalen;

            MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                deflen);

#ifdef RETRANS_SUPPORT
			if(retrans_xmit_enable == OPTION_ENABLE){
	  		  if(buff_hdr->proc_blkno != 0){
				/*acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;*/
				acq_tbl->proc_prod_bytes_read += datalen;
			  }
	  		}
#endif
        }

        pfrag->recsiz = deflen;
        heapcount += deflen;

        if (prod.head == NULL) {
            prod.head = pfrag;
            prod.tail = pfrag;
        }
        else {
            prod.tail->next = pfrag;
            prod.tail = pfrag;
        }
 
#ifdef RETRANS_SUPPORT
		 if(((prod.nfrag == 0) || (prod.nfrag >= (pfrag->fragnum +1))) && (save_prod == 0)){
		   if(ulogIsVerbose())
			  uinfo("Do not save prod [seqno=%ld] as its retrans dup fragnum/total fragments =[%d of %d] save_prod=[%d] \n",
			   prod.seqno,pfrag->fragnum,prod.nfrag,save_prod);
		   ds_free ();
		   prod.head = NULL;
		   prod.tail = NULL;
		   PNGINIT = 0;
		}else{
#endif
          if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
            if (GOES == 1) {
                if (PNGINIT == 1) {
                    pngout_end();
                    heapcount = png_get_prodlen();
                }
                else {
                    if (ulogIsDebug())
                        udebug("GOES product already compressed %d", heapcount);
                }
            }
            if (ulogIsVerbose())
              uinfo("we should have a complete product %ld %ld/%ld %ld /heap "
                  "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum,
                 (long) heapcount);
            if ((NWSTG == 1) && (heapcount > 4)) {
                cnt = 4;                /* number of bytes to add for TRAILER */

                /*
                 * Do a DDPLUS vs HDS check for NWSTG channel only
                 */
                if (sbn->datastream == 5) {
                    /* nwstg channel */
                    switch (psh->pcat) {
                    case 1:
                    case 7:
                      /* Do a quick check for non-ascii text products */
                      if (!prod_isascii(PROD_NAME, memheap, heapcount))
                        psh->pcat += 100;       /* call these HDS */
                      break;
                    }
                }

                if (cnt > 0) {
                    memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        cnt);
                    heapcount += cnt;
                }
            }

#ifdef RETRANS_SUPPORT
			if((retrans_xmit_enable == OPTION_ENABLE) && (acq_tbl->read_distrib_enable & READ_CTL_DISCARD)){
						num_prod_discards++;
						/* Set discard_prod to 1; Otherwise already stored prod may be requested for retransmit */
						discard_prod=1;
						if(ulogIsVerbose())
						  uinfo("No of products discarded = %ld prod.seqno=%ld \n ",num_prod_discards,prod.seqno);
						prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
						acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno -1 ;
						ds_free ();
						prod.head = NULL;
						prod.tail = NULL;
						PNGINIT = 0;
			}else{
				/* Do not insert prod into queue if its a duplicate product */
				if(save_prod != 0)
#endif
                                   process_prod(prod, PROD_NAME, memheap, heapcount,
                                                md5ctxp, productMaker->ldmProdQueue, psh, sbn);
#ifdef RETRANS_SUPPORT
				/* Update acq table with last processed seqno -Begin */
				if(retrans_xmit_enable == OPTION_ENABLE){
					acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
					uinfo(" prod with seqno processed = %ld\n",acq_tbl->proc_base_prod_seqno_last);
				}
				/* Update acq table with last processed seqno -End */
#endif

            ds_free();

            prod.head = NULL;
            prod.tail = NULL;
            PNGINIT = 0;

            (void)pthread_mutex_lock(&productMaker->mutex);
            productMaker->nprods++;
            (void)pthread_mutex_unlock(&productMaker->mutex);
#ifdef RETRANS_SUPPORT
        }
#endif
       }
        else {
            if (ulogIsDebug())
                udebug("processing record %ld [%ld %ld]", prod.seqno,
                    prod.nfrag, pfrag->fragnum);
            if ((pdh->transtype & 4) > 0) {
                uerror("Hmmm....should call completed product %ld [%ld %ld]",
                    prod.seqno, prod.nfrag, pfrag->fragnum);
            }
        }

#ifdef RETRANS_SUPPORT
	   if(retrans_xmit_enable == OPTION_ENABLE){
	     if(!(acq_tbl->read_distrib_enable & READ_CTL_DISCARD))
	   	  if(!discard_prod){
		    acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
		    discard_prod = 0;
	      }
	   }  
#endif
		
#ifdef RETRANS_SUPPORT
	}

        save_prod = 1;
#endif
        IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize);

        if (ulogIsDebug())
            udebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen,
                deflen);
#ifdef RETRANS_SUPPORT
		if(retrans_xmit_enable == OPTION_ENABLE){
		  total_prods_retrans_rcvd = acq_tbl->proc_tot_prods_retrans_rcvd;     /* prods retrans rcvd by proc */
		  total_prods_retrans_rcvd_lost = acq_tbl->proc_tot_prods_retrans_rcvd_lost; /* prods retrans rcvd lost */
		  total_prods_retrans_rcvd_notlost = acq_tbl->proc_tot_prods_retrans_rcvd_notlost; /* prods rcvd not lost */
		  total_prods_retrans_rqstd = acq_tbl->proc_tot_prods_retrans_rqstd;    /* prods retrans requested */
		  total_prods_handled = acq_tbl->proc_tot_prods_handled;    /* prods retrans requested */
		  total_prods_lost_err = acq_tbl->proc_tot_prods_lost_errs;    /* prods retrans requested */
		  total_frame_cnt = acq_tbl->read_tot_buff_read;
		  total_frame_err = acq_tbl->read_frame_tot_lost_errs;
		  proc_orig_prod_seqno_last_save = acq_tbl->proc_orig_prod_seqno_last;
        }
#endif
    }

    if (NULL != memheap)
        free(memheap);

    productMaker->status = status;

    return NULL;
}
Пример #2
0
/**
 * Reads NOAAPORT data from a shared-memory FIFO or a file, creates LDM
 * data-products, and inserts the data-products into an LDM product-queue.
 *
 * Usage:
 *
 *     readnoaaport [-nvx] [-q <em>queue</em>] [-u <em>n</em>] [-m mcastAddr] [path]\n
 *
 * Where:
 * <dl>
 *      <dt>-l <em>log</em></dt>
 *      <dd>Log to \e log. if \e log is "-", then logging occurs to the 
 *      standard error stream; otherwise, \e log is the pathname of a file to
 *      which logging will occur. If not specified, then log messages will go
 *      to the system logging daemon. </dd>
 *
 *      <dt>-m <em>mcastAddr</em></dt>
 *      <dd>Use the shared-memory FIFO associated with the UDP
 *      multicast address \e mcastAddr.</dd>
 *
 *      <dt>-n</dt>
 *      <dd>Log messages of level NOTICE and higher priority.</dd>
 *
 *      <dt>-q <em>queue</em></dt>
 *      <dd>Use \e queue as the pathname of the LDM product-queue. The default
 *      is to use the default LDM pathname of the product-queue.</dd>
 *
 *      <dt>-u <em>n</em></dt>
 *      <dd>If logging is to the system logging daemon, then use facility 
 *      <b>local</b><em>n</em>. The default is to use the LDM facility.
 *
 *      <dt>-v</dt>
 *      <dd>Log messages of level INFO and higher priority. Each data-product
 *      will generate a log message.</dd>
 *
 *      <dt>-x</dt>
 *      <dd>Log messages of level DEBUG and higher priority.</dd>
 *
 *      <dt><em>path</em></dt>
 *      <dd>Pathname of the file from which to read data. The default is to use
 *      a shared-memory FIFO.</dd>
 * </dl>
 *
 * @retval 0 if successful.
 * @retval 1 if an error occurred. At least one error-message is logged.
 */
int main(
     const int          argc,
     char* const        argv[])
{
#ifdef HAVE_GET_QUEUE_PATH
    const char*         pqfname = getQueuePath();
#else
    const char*         pqfname = DEFAULT_QUEUE;
#endif
    int                 fd;
    char*               prodmmap;
    char*               memheap = NULL;
    size_t              heapsize;
    size_t              heapcount;
    unsigned char       b1;
    int                 cnt, dataoff, datalen, deflen;
    int                 nscan;
    long                IOFF;
    int                 NWSTG, GOES, PNGINIT = 0, PROD_COMPRESSED;
    long                last_sbn_seqno = (-1);
    char                PROD_NAME[1024];
    int                 status;
    prodstore           prod;
    sbn_struct*         sbn;
    pdh_struct*         pdh;
    psh_struct*         psh;
    ccb_struct*         ccb;
    pdb_struct*         pdb;
    datastore*          pfrag;
    extern int          optind;
    extern int          opterr;
    extern char*        optarg;
    int                 ch;
    int                 logmask = LOG_MASK(LOG_ERR);
    const char*         logfname = NULL;        /* use system logging daemon */
    unsigned            logOptions = LOG_CONS | LOG_PID;
    unsigned            logFacility = LOG_LDM;  /* use default LDM facility */
    const char* const   progName = ubasename(argv[0]);
    MD5_CTX*            md5ctxp = NULL;
    /*unsigned char *compr;
    long                comprLen = 10000 * sizeof (int);*/
    int                 pid_channel = -1;

    /*compr = (unsigned char *) calloc (comprLen, 1);*/

    /* Initialize the logger. */
    (void)setulogmask(logmask);
    (void)openulog(progName, logOptions, logFacility, logfname);

    opterr = 1;
    while ((ch = getopt(argc, argv, "nvxl:q:u:m:")) != EOF) {
        switch (ch) {
        case 'v':
            logmask |= LOG_MASK(LOG_INFO);
            (void)setulogmask(logmask);
            break;
        case 'x':
            logmask |= LOG_MASK(LOG_DEBUG);
            (void)setulogmask(logmask);
            break;
        case 'n':
            logmask |= LOG_MASK(LOG_NOTICE);
            (void)setulogmask(logmask);
            break;
        case 'l':
            if (optarg[0] == '-' && optarg[1] != 0) {
                nplError("logfile \"%s\" ??\n", optarg);
                usage(argv[0]);
            }
            /* else */
            logfname = optarg;
            (void)openulog(progName, logOptions, logFacility, logfname);
            break;
        case 'q':
            pqfname = optarg;
            break;
        case 'u': {
            int         i = atoi(optarg);

            if (0 <= i && 7 >= i) {
                static int  logFacilities[] = {LOG_LOCAL0, LOG_LOCAL1,
                    LOG_LOCAL2, LOG_LOCAL3, LOG_LOCAL4, LOG_LOCAL5, LOG_LOCAL6,
                    LOG_LOCAL7};

                logFacility = logFacilities[i];

                (void)openulog(progName, logOptions, logFacility, logfname);
            }

            break;
        }
        case 'm':
            sscanf(optarg, "%*d.%*d.%*d.%d", &pid_channel);	
            if ((pid_channel < 1) || (pid_channel > MAX_DVBS_PID)) {
                pid_channel = -1;
            }
            else {  
                shm = shmfifo_new();
                cnt = 0;

                while (((status = shmfifo_shm_from_key(shm, 
                          s_port[pid_channel - 1])) == -3) && (cnt < 30)) {
                    nplInfo("Trying to get shared-memory FIFO");
                    cnt++;
                    sleep(1);
                }

                if (0 != status) {
                    nplError("Couldn't get shared-memory FIFO. "
                            "Check associated dvbs_multicast(1) process.");
                    shmfifo_free(shm);
                    shm = NULL;
                }
                else {
                    nplInfo("Got shared-memory FIFO");
                }
            }
            break;
        case '?':
            usage(argv[0]);
            break;
        }
    }

    if (argc - optind < 0)
        usage(argv[0]);

    nplNotice("Starting Up %s", PACKAGE_VERSION);

    fd = ((argc - optind) == 0)
        ? fileno(stdin)
        : open(argv[optind], O_RDONLY, 0);

    if ((!shm) && (fd == -1)) {
        nplError("could not open input file");
        exit(0);
    }

    /*
     * Set up signal handlers
     */
    set_sigactions();

    /*
     * Register atexit routine
     */
    if (atexit(cleanup) != 0) {
        nplSerror("atexit");
        exit(-1);
    }

    sbn = (sbn_struct*)malloc(sizeof(sbn_struct));
    pdh = (pdh_struct*)malloc(sizeof(pdh_struct));
    psh = (psh_struct*)malloc(sizeof(psh_struct));
    ccb = (ccb_struct*)malloc(sizeof(ccb_struct));
    pdb = (pdb_struct*)malloc(sizeof(pdb_struct));
    prodmmap = (char*)malloc(10000);

    if (prodmmap == NULL) {
        nplError("could not allocate read buffer");
        exit(-1);
    }

    md5ctxp = new_MD5_CTX();
    prod.head = NULL;
    prod.tail = NULL;

    if (lpqGet(pqfname, &ldmProdQueue) != 0) {
        NPL_ADD1("Couldn't open LDM product-queue \"%s\"", pqfname);
        exit(1);
    }

    while (DONE == 0) {
        /* See if any stats need to be logged */
        if (logstats) {
            logstats = 0;
            dump_stats();
        }

        /* Look for first byte == 255  and a valid SBN checksum */
        if ((status = bufread(fd, prodmmap, 1)) != 0) {
            if (-3 == status)
                break;
            abort();
        }
        if ((b1 = (unsigned char)prodmmap[0]) != 255) {
            if (ulogIsVerbose())
                nplInfo("trying to resync %u", b1);
            if (ulogIsDebug())
                nplDebug("bufread loop");
            continue;
        }

        if (bufread(fd, prodmmap + 1, 15) != 0) {
            if (ulogIsDebug())
                nplDebug("couldn't read 16 bytes for sbn");
            continue;
        }

        while ((status = readsbn(prodmmap, sbn)) != 0) {
            if (ulogIsDebug())
                nplDebug("Not SBN start");

            IOFF = 1;

            while ((IOFF < 16) && ((b1 = (unsigned char) prodmmap[IOFF]) !=
                        255))
                IOFF++;

            if (IOFF > 15) {
                break;
            }
            else {
                for (ch = IOFF; ch < 16; ch++)
                    prodmmap[ch - IOFF] = prodmmap[ch];

                if (bufread(fd, prodmmap + 16 - IOFF, IOFF) != 0) {
                    if (ulogIsDebug())
                        nplDebug("Couldn't read bytes for SBN, resync");
                    break;
                }
            }
        }

        if (status != 0) {
            if (ulogIsDebug())
                nplDebug("SBN status continue");
            continue;
        }

        IOFF = 0;

        if (bufread(fd, prodmmap + 16, 16) != 0) {
            if (ulogIsDebug())
                nplDebug("error reading Product Definition Header");
            continue;
        }

        if (ulogIsDebug())
            nplDebug("***********************************************");
        if (last_sbn_seqno != -1) {
            if (sbn->seqno != last_sbn_seqno + 1) {
                nplNotice("Gap in SBN sequence number %ld to %ld [skipped %ld]",
                         last_sbn_seqno, sbn->seqno,
                         sbn->seqno - last_sbn_seqno - 1);
                if ( sbn->seqno > last_sbn_seqno )
                    nmissed = nmissed + 
                        (unsigned long)(sbn->seqno - last_sbn_seqno - 1);
            }
        }

        last_sbn_seqno = sbn->seqno;

        if (ulogIsVerbose())
            nplInfo("SBN seqnumber %ld", sbn->seqno);
        if (ulogIsVerbose())
            nplInfo("SBN datastream %d command %d", sbn->datastream,
                sbn->command);
        if (ulogIsDebug())
            nplDebug("SBN version %d length offset %d", sbn->version, sbn->len);
        if (((sbn->command != 3) && (sbn->command != 5)) || 
                (sbn->version != 1)) {
            nplError("Unknown sbn command/version %d PUNT", sbn->command);
            continue;
        }

        switch (sbn->datastream) {
        case 7:	      /* test */
        case 6:       /* was reserved...now nwstg2 */
        case 5:
            NWSTG = 1;
            GOES = 0;
            break;
        case 1:
        case 2:
        case 4:
            NWSTG = 0;
            GOES = 1;
            break;
        default:
            nplError("Unknown NOAAport channel %d PUNT", sbn->datastream);
            continue;
        }

        /* End of SBN version low 4 bits */

        if (readpdh(prodmmap + IOFF + sbn->len, pdh) == -1) {
            nplError("problem with pdh, PUNT");
            continue;
        }
        if (pdh->len > 16) {
            bufread(fd, prodmmap + sbn->len + 16, pdh->len - 16);
        }

        if (ulogIsDebug())
            nplDebug("Product definition header version %d pdhlen %d",
                pdh->version, pdh->len);

        if (pdh->version != 1) {
            nplError("Error: PDH transfer type %u, PUNT", pdh->transtype);
            continue;
        }
        else if (ulogIsDebug()) {
            nplDebug("PDH transfer type %u", pdh->transtype);
        }

        if ((pdh->transtype & 8) > 0)
            nplError("Product transfer flag error %u", pdh->transtype);
        if ((pdh->transtype & 32) > 0)
            nplError("Product transfer flag error %u", pdh->transtype);

        if ((pdh->transtype & 16) > 0) {
            PROD_COMPRESSED = 1;

            if (ulogIsDebug())
                nplDebug("Product transfer flag compressed %u", pdh->transtype);
        }
        else {
            PROD_COMPRESSED = 0;
        }

        if (ulogIsDebug())
            nplDebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen,
                pdh->pshlen);
        if (ulogIsDebug())
            nplDebug("blocks per record %ld records per block %ld\n",
                pdh->blocks_per_record, pdh->records_per_block);
        if (ulogIsDebug())
            nplDebug("product seqnumber %ld block number %ld data block size "
                "%ld", pdh->seqno, pdh->dbno, pdh->dbsize);

        /* Stop here if no psh */
        if ((pdh->pshlen == 0) && (pdh->transtype == 0)) {
            IOFF = IOFF + sbn->len + pdh->len;
            continue;
        }

        if (pdh->pshlen != 0) {
            if (bufread(fd, prodmmap + sbn->len + pdh->len, pdh->pshlen) != 0) {
                nplError("problem reading psh");
                continue;
            }
            else {
                if (ulogIsDebug())
                    nplDebug("read psh %d", pdh->pshlen);
            }

            /* Timing block */
            if (sbn->command == 5) {
                if (ulogIsDebug())
                    nplDebug("Timing block recieved %ld %ld\0", psh->olen,
                        pdh->len);
                /*
                 * Don't step on our psh of a product struct of prod in
                 * progress.
                 */
                continue;
            }

            if (readpsh(prodmmap + IOFF + sbn->len + pdh->len, psh) == -1) {
                nplError("problem with readpsh");
                continue;
            }
            if (psh->olen != pdh->pshlen) {
                nplError("ERROR in calculation of psh len %ld %ld", psh->olen,
                    pdh->len);
                continue;
            }
            if (ulogIsDebug())
                nplDebug("len %ld", psh->olen);
            if (ulogIsDebug())
                nplDebug("product header flag %d, version %d", psh->hflag,
                    psh->version);
            if (ulogIsDebug())
                nplDebug("prodspecific data length %ld", psh->psdl);
            if (ulogIsDebug())
                nplDebug("bytes per record %ld", psh->bytes_per_record);
            if (ulogIsDebug())
                nplDebug("Fragments = %ld category %d ptype %d code %d",
                    psh->frags, psh->pcat, psh->ptype, psh->pcode);
            if (psh->frags < 0)
                nplError("check psh->frags %d", psh->frags);
            if (psh->origrunid != 0)
                nplError("original runid %d", psh->origrunid);
            if (ulogIsDebug())
                nplDebug("next header offset %ld", psh->nhoff);
            if (ulogIsDebug())
                nplDebug("original seq number %ld", psh->seqno);
            if (ulogIsDebug())
                nplDebug("receive time %ld", psh->rectime);
            if (ulogIsDebug())
                nplDebug("transmit time %ld", psh->transtime);
            if (ulogIsDebug())
                nplDebug("run ID %ld", psh->runid);
            if (ulogIsDebug())
                nplDebug("original run id %ld", psh->origrunid);
            if (prod.head != NULL) {
                nplError("OOPS, start of new product [%ld ] with unfinished "
                    "product %ld", pdh->seqno, prod.seqno);

                ds_free();

                prod.head = NULL;
                prod.tail = NULL;

                if (PNGINIT != 0) {
                    pngout_end();
                    PNGINIT = 0;
                }

                nplError("Product definition header version %d pdhlen %d",
                        pdh->version, pdh->len);
                nplError("PDH transfer type %u", pdh->transtype);

                if ((pdh->transtype & 8) > 0)
                    nplError("Product transfer flag error %u", pdh->transtype);
                if ((pdh->transtype & 32) > 0)
                    nplError("Product transfer flag error %u", pdh->transtype);

                nplError("header length %ld [pshlen = %d]",
                    pdh->len + pdh->pshlen, pdh->pshlen);
                nplError("blocks per record %ld records per block %ld",
                    pdh->blocks_per_record, pdh->records_per_block);
                nplError("product seqnumber %ld block number %ld data block "
                    "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize);
                nplError("product header flag %d", psh->hflag);
                nplError("prodspecific data length %ld", psh->psdl);
                nplError("bytes per record %ld", psh->bytes_per_record);
                nplError("Fragments = %ld category %d", psh->frags, psh->pcat);

                if (psh->frags < 0)
                    nplError("check psh->frags %d", psh->frags);
                if (psh->origrunid != 0)
                    nplError("original runid %d", psh->origrunid);

                nplError("next header offset %ld", psh->nhoff);
                nplError("original seq number %ld", psh->seqno);
                nplError("receive time %ld", psh->rectime);
                nplError("transmit time %ld", psh->transtime);
                nplError("run ID %ld", psh->runid);
                nplError("original run id %ld", psh->origrunid);
            }

            prod.seqno = pdh->seqno;
            prod.nfrag = psh->frags;

            ds_init(prod.nfrag);

            /* NWSTG CCB = dataoff, WMO = dataoff + 24 */

            if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen,
                    pdh->dbsize) != 0) {
                nplError("problem reading datablock");
                continue;
            }
            if (sbn->datastream == 4) {
                if (psh->pcat != 3) {
                    GOES = 0;
                    NWSTG = 1;
                }
            }

            heapcount = 0;

            MD5Init(md5ctxp);

            if (GOES == 1) {
                if (readpdb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen,
                        psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) {
                    nplError("Error reading pdb, punt");
                    continue;
                }

                memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                if (ulogIsDebug())
                    nplDebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len,
                        pdh->pshlen, sbn->len + pdh->len + pdh->pshlen,
                        pdb->len);

                /* Data starts at first block after pdb */
                ccb->len = 0;
                heapsize = prodalloc(psh->frags, 5152, &memheap);
            }
            if (NWSTG == 1) {
                memset(psh->pname, 0, sizeof(psh->pname));

                if (readccb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen,
                        ccb, psh, pdh->dbsize) == -1)
                    nplError("Error reading ccb, using default name");
                if (ulogIsDebug())
                    nplDebug("look at ccb start %d %d", ccb->b1, ccb->len);

                /*
                   cnt = 0;
                   memset(psh->pname,0,sizeof(psh->pname));
                   while ((b1 = (unsigned char)prodmmap[
                           IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len +
                           cnt]) >= 32) {
                       psh->pname[cnt] = prodmmap[
                           IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len +
                           cnt];
                       cnt++;
                   } 
                   if(cnt > 0)
                 */
                if (ulogIsVerbose())
                    nplInfo("%s", psh->pname);

                memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                heapsize = prodalloc(psh->frags, 4000 + 15, &memheap);
                /*
                 * We will only compute md5 checksum on the data, 11 FOS
                 * characters at start
                 */
                /*
                 * sprintf(memheap,"\001\015\015\012%04d\015\015\012",
                 * ((int)pdh->seqno)%10000);
                 */
                sprintf(memheap, "\001\015\015\012%03d\040\015\015\012",
                    ((int) pdh->seqno) % 1000);

                heapcount += 11;

                if (psh->metaoff > 0)
                    psh->metaoff = psh->metaoff + 11;
            }
        }
        else {
            /* If a continuation record...don't let psh->pcat get missed */
            if ((sbn->datastream == 4) && (psh->pcat != 3)) {
                GOES = 0;
                NWSTG = 1;
            }

            ccb->len = 0;

            if (ulogIsDebug())
                nplDebug("continuation record");
            if ((pdh->transtype & 4) > 0) {
                psh->frags = 0;
            }
            if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen,
                    pdh->dbsize) != 0) {
                nplError("problem reading datablock (cont)");
                continue;
            }
            if (prod.head == NULL) {
                if (ulogIsVerbose())
                    nplInfo("found data block before header, "
                        "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno);
                continue;
            }
        }

        /* Get the data */
        dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len;
        datalen = pdh->dbsize - ccb->len;

        if (ulogIsDebug())
            nplDebug("look at datalen %d", datalen);

        pfrag = ds_alloc();
        pfrag->seqno = pdh->seqno;
        pfrag->fragnum = pdh->dbno;
        pfrag->recsiz = datalen;
        pfrag->offset = heapcount;
        pfrag->next = NULL;

        /*memcpy(memheap+heapcount,prodmmap+dataoff,datalen);
        MD5Update(md5ctxp, (unsigned char *)(memheap+heapcount), datalen);
        test_deflate(compr,comprLen,(unsigned char *)(memheap+heapcount),
        datalen);*/

        if (GOES == 1) {
            if (pfrag->fragnum > 0) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) || 
                        (pfrag->seqno != prod.seqno)) {
                    nplError("Missing GOES fragment in sequence, "
                        "last %d/%d this %d/%d\0", prod.tail->fragnum,
                        prod.seqno, pfrag->fragnum, pfrag->seqno);
                    ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }

                if ((PNGINIT != 1) && (!PROD_COMPRESSED)) {
                    nplError("failed pnginit %d %d %s", sbn->datastream,
                            psh->pcat, PROD_NAME);
                    continue;
                }
                if (pdh->records_per_block < 1) {
                    nplError("records_per_block %d blocks_per_record %d "
                        "nx %d ny %d", pdh->records_per_block,
                        pdh->blocks_per_record, pdb->nx, pdb->ny);
                    nplError("source %d sector %d channel %d", pdb->source,
                        pdb->sector, pdb->channel);
                    nplError("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                        "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year,
                        pdb->month, pdb->day, pdb->hour, pdb->minute,
                        pdb->second, pdb->sechunds);
                    nplError("pshname %s", psh->pname);
                }
                if (!PROD_COMPRESSED) {
                    for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) {
                        if (ulogIsDebug())
                            nplDebug("png write nscan %d", nscan);
                        if (nscan >= pdh->records_per_block) {
                            nplError("nscan exceeding records per block %d [%d "
                                "%d %d]", pdh->records_per_block, nscan,
                                pdb->nx, pdh->dbsize);
                        }
                        else {
                          pngwrite(prodmmap + dataoff + (nscan * pdb->nx));
                        }
                    }
                }
                else {
                    memcpy(memheap + heapcount, prodmmap + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
            }
            else {
                if (!PROD_COMPRESSED) {
                    png_set_memheap(memheap, md5ctxp);
                    png_header(prodmmap + dataoff, datalen);
                    /*
                     * Add 1 to number of scanlines, image ends with 
                     * f0f0f0f0...
                     */
                    pngout_init(pdb->nx, pdb->ny + 1);

                    PNGINIT = 1;
                }
                else {
                    memcpy(memheap + heapcount, prodmmap + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
                nplNotice("records_per_block %d blocks_per_record %d nx %d ny %d",
                    pdh->records_per_block, pdh->blocks_per_record, pdb->nx,
                    pdb->ny);
                nplNotice("source %d sector %d channel %d", pdb->source,
                    pdb->sector, pdb->channel);
                nplNotice("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                    "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month,
                    pdb->day, pdb->hour, pdb->minute, pdb->second,
                    pdb->sechunds);
                nplNotice("pshname %s", psh->pname);
            }
            deflen = 0;
        }
        else {
            /*
             * test_deflate(memheap+heapcount,heapsize-heapcount,(unsigned char
             * *)(prodmmap+dataoff),datalen,&deflen);
             */
            /* If the product already has a FOS trailer, don't add
             * another....this will match what pqing(SDI) sees
             */
            if ((prod.nfrag != 0) && (prod.tail != NULL)) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) ||
                        (pfrag->seqno != prod.seqno)) {
                    nplError("Missing fragment in sequence, last %d/%d this "
                        "%d/%d\0", prod.tail->fragnum, prod.seqno,
                        pfrag->fragnum, pfrag->seqno);
                    ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }
            }
            if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
                char testme[4];

                while (datalen > 4) {
                    memcpy(testme, prodmmap + (dataoff + datalen - 4), 4);

                    if (memcmp(testme, FOS_TRAILER, 4) == 0) {
                        datalen -= 4;

                        if (ulogIsDebug())
                            nplDebug("removing FOS trailer from %s", PROD_NAME);
                    }
                    else {
                        break;
                    }
                }
            }
            if (heapcount + datalen > heapsize) {
                /*
                 * this above wasn't big enough heapsize =
                 * prodalloc(psh->frags,4000+15,&memheap);
                 */
                nplError("Error in heapsize %d product size %d [%d %d], Punt!\0",
                    heapsize, (heapcount + datalen), heapcount, datalen);
                continue;
            }

            memcpy(memheap + heapcount, prodmmap + dataoff, datalen);

            deflen = datalen;

            MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                deflen);
        }

        pfrag->recsiz = deflen;
        /*heapcount += datalen;*/
        heapcount += deflen;

        if (prod.head == NULL) {
            prod.head = pfrag;
            prod.tail = pfrag;
        }
        else {
            prod.tail->next = pfrag;
            prod.tail = pfrag;
        }

        if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
            if (GOES == 1) {
                if (PNGINIT == 1) {
                    pngout_end();
                    heapcount = png_get_prodlen();
                }
                else {
                    if (ulogIsDebug())
                        nplDebug("GOES product already compressed %d", heapcount);
                }
            }
            if (ulogIsVerbose())
              nplInfo("we should have a complete product %ld %ld/%ld %ld /heap "
                  "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum,
                 (long) heapcount);
            if ((NWSTG == 1) && (heapcount > 4)) {
                cnt = 4;		/* number of bytes to add for TRAILER */

                /*
                 * Do a DDPLUS vs HDS check for NWSTG channel only
                 */
                if (sbn->datastream == 5) {
                    /* nwstg channel */
                    switch (psh->pcat) {
                    case 1:
                    case 7:
                      /* Do a quick check for non-ascii text products */
                      if (!prod_isascii(PROD_NAME, memheap, heapcount))
                        psh->pcat += 100;	/* call these HDS */
                      /* else {
                         ** call these DDPLUS **
                         if (memheap[heapcount-1] == FOS_TRAILER[3]) **
                             ETX check **
                             cnt = 0; ** no need to add extra ETX pqing doesn't
                             see it **
                         }
                       */
                      break;
                    }
                }

                if (cnt > 0) {
                    memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        cnt);
                    heapcount += cnt;
                }
            }

            process_prod(prod, PROD_NAME, memheap, heapcount,
                md5ctxp, ldmProdQueue, psh, sbn);
            ds_free();

            prod.head = NULL;
            prod.tail = NULL;
            PNGINIT = 0;
        }
        else {
            if (ulogIsDebug())
                nplDebug("processing record %ld [%ld %ld]", prod.seqno,
                    prod.nfrag, pfrag->fragnum);
            if ((pdh->transtype & 4) > 0) {
                nplError("Hmmm....should call completed product %ld [%ld %ld]",
                    prod.seqno, prod.nfrag, pfrag->fragnum);
            }
        }

        IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize);

        if (ulogIsDebug())
            nplDebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen,
                deflen);
    }

    if (fd != -1)
       (void)close(fd);

    exit(0);
}