/*************************************************************************** * msr_pack: * * Pack data into SEED data records. Using the record header values * in the MSRecord as a template the common header fields are packed * into the record header, blockettes in the blockettes chain are * packed and data samples are packed in the encoding format indicated * by the MSRecord->encoding field. A Blockette 1000 will be added if * one is not present. * * The MSRecord->datasamples array and MSRecord->numsamples value will * not be changed by this routine. It is the responsibility of the * calling routine to adjust the data buffer if desired. * * As each record is filled and finished they are passed to * record_handler which expects 1) a char * to the record, 2) the * length of the record and 3) a pointer supplied by the original * caller containing optional private data (handlerdata). It is the * responsibility of record_handler to process the record, the memory * will be re-used or freed when record_handler returns. * * If the flush flag != 0 all of the data will be packed into data * records even though the last one will probably not be filled. * * Default values are: data record & quality indicator = 'D', record * length = 4096, encoding = 11 (Steim2) and byteorder = 1 (MSBF). * The defaults are triggered when the the msr->dataquality is 0 or * msr->reclen, msr->encoding and msr->byteorder are -1 respectively. * * Returns the number of records created on success and -1 on error. ***************************************************************************/ int msr_pack ( MSRecord * msr, void (*record_handler) (char *, int, void *), void *handlerdata, int64_t *packedsamples, flag flush, flag verbose ) { uint16_t *HPnumsamples; uint16_t *HPdataoffset; struct blkt_1001_s *HPblkt1001 = NULL; char *rawrec; char *envvariable; char srcname[50]; flag headerswapflag = 0; flag dataswapflag = 0; flag packret; int samplesize; int headerlen; int dataoffset; int maxdatabytes; int maxsamples; int recordcnt = 0; int packsamples, packoffset; int64_t totalpackedsamples; hptime_t segstarttime; if ( ! msr ) return -1; if ( ! record_handler ) { ms_log (2, "msr_pack(): record_handler() function pointer not set!\n"); return -1; } /* Allocate stream processing state space if needed */ if ( ! msr->ststate ) { msr->ststate = (StreamState *) malloc (sizeof(StreamState)); if ( ! msr->ststate ) { ms_log (2, "msr_pack(): Could not allocate memory for StreamState\n"); return -1; } memset (msr->ststate, 0, sizeof(StreamState)); } /* Generate source name for MSRecord */ if ( msr_srcname (msr, srcname, 1) == NULL ) { ms_log (2, "msr_unpack_data(): Cannot generate srcname\n"); return MS_GENERROR; } /* Set shared srcname pointer to source name */ PACK_SRCNAME = &srcname[0]; /* Track original segment start time for new start time calculation */ segstarttime = msr->starttime; /* Read possible environmental variables that force byteorder */ if ( packheaderbyteorder == -2 ) { if ( (envvariable = getenv("PACK_HEADER_BYTEORDER")) ) { if ( *envvariable != '0' && *envvariable != '1' ) { ms_log (2, "Environment variable PACK_HEADER_BYTEORDER must be set to '0' or '1'\n"); return -1; } else if ( *envvariable == '0' ) { packheaderbyteorder = 0; if ( verbose > 2 ) ms_log (1, "PACK_HEADER_BYTEORDER=0, packing little-endian header\n"); } else { packheaderbyteorder = 1; if ( verbose > 2 ) ms_log (1, "PACK_HEADER_BYTEORDER=1, packing big-endian header\n"); } } else { packheaderbyteorder = -1; } } if ( packdatabyteorder == -2 ) { if ( (envvariable = getenv("PACK_DATA_BYTEORDER")) ) { if ( *envvariable != '0' && *envvariable != '1' ) { ms_log (2, "Environment variable PACK_DATA_BYTEORDER must be set to '0' or '1'\n"); return -1; } else if ( *envvariable == '0' ) { packdatabyteorder = 0; if ( verbose > 2 ) ms_log (1, "PACK_DATA_BYTEORDER=0, packing little-endian data samples\n"); } else { packdatabyteorder = 1; if ( verbose > 2 ) ms_log (1, "PACK_DATA_BYTEORDER=1, packing big-endian data samples\n"); } } else { packdatabyteorder = -1; } } /* Set default indicator, record length, byte order and encoding if needed */ if ( msr->dataquality == 0 ) msr->dataquality = 'D'; if ( msr->reclen == -1 ) msr->reclen = 4096; if ( msr->byteorder == -1 ) msr->byteorder = 1; if ( msr->encoding == -1 ) msr->encoding = DE_STEIM2; /* Cleanup/reset sequence number */ if ( msr->sequence_number <= 0 || msr->sequence_number > 999999) msr->sequence_number = 1; if ( msr->reclen < MINRECLEN || msr->reclen > MAXRECLEN ) { ms_log (2, "msr_pack(%s): Record length is out of range: %d\n", PACK_SRCNAME, msr->reclen); return -1; } if ( msr->numsamples <= 0 ) { ms_log (2, "msr_pack(%s): No samples to pack\n", PACK_SRCNAME); return -1; } samplesize = ms_samplesize (msr->sampletype); if ( ! samplesize ) { ms_log (2, "msr_pack(%s): Unknown sample type '%c'\n", PACK_SRCNAME, msr->sampletype); return -1; } /* Sanity check for msr/quality indicator */ if ( ! MS_ISDATAINDICATOR(msr->dataquality) ) { ms_log (2, "msr_pack(%s): Record header & quality indicator unrecognized: '%c'\n", PACK_SRCNAME, msr->dataquality); ms_log (2, "msr_pack(%s): Packing failed.\n", PACK_SRCNAME); return -1; } /* Allocate space for data record */ rawrec = (char *) malloc (msr->reclen); if ( rawrec == NULL ) { ms_log (2, "msr_pack(%s): Cannot allocate memory\n", PACK_SRCNAME); return -1; } /* Set header pointers to known offsets into FSDH */ HPnumsamples = (uint16_t *) (rawrec + 30); HPdataoffset = (uint16_t *) (rawrec + 44); /* Check to see if byte swapping is needed */ if ( msr->byteorder != ms_bigendianhost() ) headerswapflag = dataswapflag = 1; /* Check if byte order is forced */ if ( packheaderbyteorder >= 0 ) { headerswapflag = ( msr->byteorder != packheaderbyteorder ) ? 1 : 0; } if ( packdatabyteorder >= 0 ) { dataswapflag = ( msr->byteorder != packdatabyteorder ) ? 1 : 0; } if ( verbose > 2 ) { if ( headerswapflag && dataswapflag ) ms_log (1, "%s: Byte swapping needed for packing of header and data samples\n", PACK_SRCNAME); else if ( headerswapflag ) ms_log (1, "%s: Byte swapping needed for packing of header\n", PACK_SRCNAME); else if ( dataswapflag ) ms_log (1, "%s: Byte swapping needed for packing of data samples\n", PACK_SRCNAME); else ms_log (1, "%s: Byte swapping NOT needed for packing\n", PACK_SRCNAME); } /* Add a blank 1000 Blockette if one is not present, the blockette values will be populated in msr_pack_header_raw()/msr_normalize_header() */ if ( ! msr->Blkt1000 ) { struct blkt_1000_s blkt1000; memset (&blkt1000, 0, sizeof (struct blkt_1000_s)); if ( verbose > 2 ) ms_log (1, "%s: Adding 1000 Blockette\n", PACK_SRCNAME); if ( ! msr_addblockette (msr, (char *) &blkt1000, sizeof(struct blkt_1000_s), 1000, 0) ) { ms_log (2, "msr_pack(%s): Error adding 1000 Blockette\n", PACK_SRCNAME); return -1; } } headerlen = msr_pack_header_raw (msr, rawrec, msr->reclen, headerswapflag, 1, &HPblkt1001, verbose); if ( headerlen == -1 ) { ms_log (2, "msr_pack(%s): Error packing header\n", PACK_SRCNAME); return -1; } /* Determine offset to encoded data */ if ( msr->encoding == DE_STEIM1 || msr->encoding == DE_STEIM2 ) { dataoffset = 64; while ( dataoffset < headerlen ) dataoffset += 64; /* Zero memory between blockettes and data if any */ memset (rawrec + headerlen, 0, dataoffset - headerlen); } else { dataoffset = headerlen; } *HPdataoffset = (uint16_t) dataoffset; if ( headerswapflag ) ms_gswap2 (HPdataoffset); /* Determine the max data bytes and sample count */ maxdatabytes = msr->reclen - dataoffset; if ( msr->encoding == DE_STEIM1 ) { maxsamples = (int) (maxdatabytes/64) * STEIM1_FRAME_MAX_SAMPLES; } else if ( msr->encoding == DE_STEIM2 ) { maxsamples = (int) (maxdatabytes/64) * STEIM2_FRAME_MAX_SAMPLES; } else { maxsamples = maxdatabytes / samplesize; } /* Pack samples into records */ *HPnumsamples = 0; totalpackedsamples = 0; if ( packedsamples ) *packedsamples = 0; packoffset = 0; while ( (msr->numsamples - totalpackedsamples) > maxsamples || flush ) { packret = msr_pack_data (rawrec + dataoffset, (char *) msr->datasamples + packoffset, (int)(msr->numsamples - totalpackedsamples), maxdatabytes, &packsamples, &msr->ststate->lastintsample, msr->ststate->comphistory, msr->sampletype, msr->encoding, dataswapflag, verbose); if ( packret ) { ms_log (2, "msr_pack(%s): Error packing record\n", PACK_SRCNAME); return -1; } packoffset += packsamples * samplesize; /* Update number of samples */ *HPnumsamples = (uint16_t) packsamples; if ( headerswapflag ) ms_gswap2 (HPnumsamples); if ( verbose > 0 ) ms_log (1, "%s: Packed %d samples\n", PACK_SRCNAME, packsamples); /* Send record to handler */ record_handler (rawrec, msr->reclen, handlerdata); totalpackedsamples += packsamples; if ( packedsamples ) *packedsamples = totalpackedsamples; msr->ststate->packedsamples += packsamples; /* Update record header for next record */ msr->sequence_number = ( msr->sequence_number >= 999999 ) ? 1 : msr->sequence_number + 1; if ( msr->samprate > 0 ) msr->starttime = segstarttime + (hptime_t)(totalpackedsamples / msr->samprate * HPTMODULUS + 0.5); msr_update_header (msr, rawrec, headerswapflag, HPblkt1001, verbose); recordcnt++; msr->ststate->packedrecords++; /* Set compression history flag for subsequent records (Steim encodings) */ if ( ! msr->ststate->comphistory ) msr->ststate->comphistory = 1; if ( totalpackedsamples >= msr->numsamples ) break; } if ( verbose > 2 ) ms_log (1, "%s: Packed %d total samples\n", PACK_SRCNAME, totalpackedsamples); free (rawrec); return recordcnt; } /* End of msr_pack() */
/*************************************************************************** * msr_duplicate: * * Duplicate an MSRecord struct * including the fixed-section data * header and blockette chain. If * the datadup flag is true and the * source MSRecord has associated * data samples copy them as well. * * Returns a pointer to a new MSRecord on success and NULL on error. ***************************************************************************/ MSRecord * msr_duplicate (MSRecord *msr, flag datadup) { MSRecord *dupmsr = 0; int samplesize = 0; if ( ! msr ) return NULL; /* Allocate target MSRecord structure */ if ( (dupmsr = msr_init (NULL)) == NULL ) return NULL; /* Copy MSRecord structure */ memcpy (dupmsr, msr, sizeof(MSRecord)); /* Copy fixed-section data header structure */ if ( msr->fsdh ) { /* Allocate memory for new FSDH structure */ if ( (dupmsr->fsdh = (struct fsdh_s *) malloc (sizeof(struct fsdh_s))) == NULL ) { ms_log (2, "msr_duplicate(): Error allocating memory\n"); free (dupmsr); return NULL; } /* Copy the contents */ memcpy (dupmsr->fsdh, msr->fsdh, sizeof(struct fsdh_s)); } /* Copy the blockette chain */ if ( msr->blkts ) { BlktLink *blkt = msr->blkts; BlktLink *next = NULL; dupmsr->blkts = 0; while ( blkt ) { next = blkt->next; /* Add blockette to chain of new MSRecord */ if ( msr_addblockette (dupmsr, blkt->blktdata, blkt->blktdatalen, blkt->blkt_type, 0) == NULL ) { ms_log (2, "msr_duplicate(): Error adding blockettes\n"); msr_free (&dupmsr); return NULL; } blkt = next; } } /* Copy data samples if requested and available */ if ( datadup && msr->datasamples ) { /* Determine size of samples in bytes */ samplesize = ms_samplesize (msr->sampletype); if ( samplesize == 0 ) { ms_log (2, "msr_duplicate(): unrecognized sample type: '%c'\n", msr->sampletype); free (dupmsr); return NULL; } /* Allocate memory for new data array */ if ( (dupmsr->datasamples = (void *) malloc ((size_t)(msr->numsamples * samplesize))) == NULL ) { ms_log (2, "msr_duplicate(): Error allocating memory\n"); free (dupmsr); return NULL; } /* Copy the data array */ memcpy (dupmsr->datasamples, msr->datasamples, ((size_t)(msr->numsamples * samplesize))); } /* Otherwise make sure the sample array and count are zero */ else { dupmsr->datasamples = 0; dupmsr->numsamples = 0; } return dupmsr; } /* End of msr_duplicate() */