Example #1
0
void
readcheck(
	void *buffer,
	size_t size,
	int msgNum)
{
	size_t ngot;

	if (((ngot = buffered_read(CSD_fd, buffer, size)) != size) &&
	    ngot != (size_t)-1) {
		char *explanation;

		explanation = catgets(catfd, SET, msgNum, "??");
		error(1, errno,
		    catgets(catfd, SET, 13502,
		    "Corrupt samfsdump file.  read(%d) returned %d bytes, %s"),
		    size, ngot, explanation);
	} else if (ngot == (size_t)-1) {
		char *explanation;

		explanation = catgets(catfd, SET, msgNum, "??");
		error(1, errno,
		    catgets(catfd, SET, 13503,
		    "System error return from read of samfsdump file, %s"),
		    explanation);
	}
}
Example #2
0
// get_sized_buffer will alloc and set a buffer in the passed wtv_chunked_buffer struct
// it will handle any meta data chunks that need to be skipped in the file
// Will print error messages and return a null buffer on error.
void get_sized_buffer(struct ccx_demuxer *ctx, struct wtv_chunked_buffer *cb, uint32_t size)
{
	if(cb->buffer != NULL && cb->buffer_size > 0)
	{
		free(cb->buffer);
	}

	if(size > WTV_MAX_ALLOC)
	{
		mprint("\nRequested buffer of %i > %i bytes (WTV_MAX_ALLOC)!\n", size, WTV_MAX_ALLOC);
		cb->buffer = NULL;
		return;
	}

	cb->buffer = (uint8_t*)malloc(size);
	cb->buffer_size = size;
	uint64_t start = cb->filepos;

	if(cb->skip_chunks[cb->chunk] != -1 && (start + size) > cb->skip_chunks[cb->chunk])
	{
		buffered_read(ctx, cb->buffer, (int)(cb->skip_chunks[cb->chunk]-start));
		cb->filepos += cb->skip_chunks[cb->chunk]-start;
		buffered_seek(ctx, WTV_META_CHUNK_SIZE);
		cb->filepos += WTV_META_CHUNK_SIZE;
		buffered_read(ctx, cb->buffer+(cb->skip_chunks[cb->chunk]-start), (int)(size-(cb->skip_chunks[cb->chunk]-start)));
		cb->filepos += size-(cb->skip_chunks[cb->chunk]-start);
		cb->chunk++;
	}
	else
	{
		buffered_read(ctx, cb->buffer, size);
		cb->filepos += size;
		if(result != size)
		{
			free(cb->buffer);
			cb->buffer_size = 0;
			ctx->past       = cb->filepos;
			cb->buffer      = NULL;
			mprint("\nPremature end of file!\n");
			return;
		}
	}
	ctx->past = cb->filepos;
	return;
}
Example #3
0
// Read the actual values of the passed lookup offset and add them to
// the list of chunks to skip as nessasary. Returns false on error.
int add_skip_chunks(struct ccx_demuxer *ctx, struct wtv_chunked_buffer *cb, uint32_t offset, uint32_t flag)
{

	uint64_t start = ctx->past;
	buffered_seek(ctx, (int)((offset*WTV_CHUNK_SIZE) - start));
	uint64_t seek_back=0-((offset*WTV_CHUNK_SIZE)-start);

	uint32_t value;
	buffered_read(ctx, (unsigned char*)&value, 4);
	if(result!=4)
		return 0;
	seek_back-=4;
	while(value!=0)
	{
		dbg_print(CCX_DMT_PARSE, "value: %llx\n", get_meta_chunk_start(value));
		buffered_read(ctx, (unsigned char*)&value, 4);
		if(result!=4)
			return 0;
		add_chunk(cb, get_meta_chunk_start(value));
		seek_back-=4;
	}
	buffered_seek(ctx, (int)seek_back);
	return 1;
}
Example #4
0
LLONG get_data(struct lib_ccx_ctx *ctx, struct wtv_chunked_buffer *cb, struct demuxer_data *data)
{
	static int video_streams[32];
	static int alt_stream; //Stream to use for timestamps if the cc stream has broken timestamps
	static int use_alt_stream = 0;
	static int num_streams = 0;
	struct lib_cc_decode *dec_ctx = update_decoder_list(ctx);

	while(1)
	{
		int bytesread = 0;
		// Read the 32 bytes containing the GUID and length and stream_id info
		get_sized_buffer(ctx->demux_ctx, cb, 32);
		if(cb->buffer == NULL)
			return CCX_EOF;

		uint8_t guid[16];
		memcpy(&guid, cb->buffer, 16); // Read the GUID
		int x;
		for(x=0; x<16; x++)
			dbg_print(CCX_DMT_PARSE, "%02X ", guid[x]);
		dbg_print(CCX_DMT_PARSE, "\n");
		uint32_t len;
		memcpy(&len, cb->buffer+16, 4); // Read the length
		len-=32;
		dbg_print(CCX_DMT_PARSE, "len %X\n", len);
		uint32_t pad;
		pad = len%8==0 ? 0 : 8- (len % 8);  // Calculate the padding to add to the length
		// to get to the next GUID
		dbg_print(CCX_DMT_PARSE, "pad %X\n", pad);
		uint32_t stream_id;
		memcpy(&stream_id, cb->buffer+20, 4);
		stream_id = stream_id & 0x7f;       // Read and calculate the stream_id
		dbg_print(CCX_DMT_PARSE, "stream_id: 0x%X\n", stream_id);

		for(x=0; x<num_streams; x++)
			dbg_print(CCX_DMT_PARSE, "video stream_id: %X\n", video_streams[x]);
		if( !memcmp(guid, WTV_EOF, 16 ))
		{
			// This is the end of the data in this file
			// read the padding at the end of the file
			// and return one more byte
			dbg_print(CCX_DMT_PARSE, "WTV EOF\n");
			uint8_t *parsebuf;
			parsebuf = (uint8_t*)malloc(1024);
			do {
				buffered_read(ctx->demux_ctx, parsebuf, 1024);
				ctx->demux_ctx->past+=1024;
			} while (result==1024);
			ctx->demux_ctx->past+=result;
			free(parsebuf);
			free(cb->buffer);
			cb->buffer=NULL;
			//return one more byte so the final percentage is shown correctly
			*(data->buffer+data->len)=0x00;
			data->len++;
			return CCX_EOF;
		}
		if( !memcmp(guid, WTV_STREAM2, 16 ) )
		{
			// The WTV_STREAM2 GUID appares near the start of the data dir
			// It maps stream_ids to the type of stream
			dbg_print(CCX_DMT_PARSE, "WTV STREAM2\n");
			get_sized_buffer(ctx->demux_ctx, cb, 0xc+16);
			if(cb->buffer==NULL)
				return CCX_EOF;
			static unsigned char stream_type[16];
			memcpy(&stream_type, cb->buffer+0xc, 16); //Read the stream type GUID
			const void *stream_guid;
			if(ccx_options.wtvmpeg2)
				stream_guid = WTV_STREAM_VIDEO; // We want mpeg2 data if the user set -wtvmpeg2
			else
				stream_guid = WTV_STREAM_MSTVCAPTION; // Otherwise we want the MSTV captions stream
			if(!memcmp(stream_type, stream_guid, 16 ) )
			{
				video_streams[num_streams]=stream_id; // We keep a list of stream ids
				num_streams++;                        // Even though there should only be 1
			}
			if (memcmp(stream_type, WTV_STREAM_AUDIO, 16))
				alt_stream = stream_id;
			len-=28;
		}
		if (!memcmp(guid, WTV_TIMING, 16) && ((use_alt_stream < WTV_CC_TIMESTAMP_MAGIC_THRESH && check_stream_id(stream_id, video_streams, num_streams))
					|| (use_alt_stream == WTV_CC_TIMESTAMP_MAGIC_THRESH && stream_id == alt_stream)))
		{
			// The WTV_TIMING GUID contains a timestamp for the given stream_id
			dbg_print(CCX_DMT_PARSE, "WTV TIMING\n");
			get_sized_buffer(ctx->demux_ctx, cb, 0x8+0x8);
			if(cb->buffer==NULL)
				return CCX_EOF;
			int64_t time;
			memcpy(&time, cb->buffer+0x8, 8); // Read the timestamp
			dbg_print(CCX_DMT_PARSE, "TIME: %ld\n", time);
			if (time != -1 && time != WTV_CC_TIMESTAMP_MAGIC) { // Ignore -1 timestamps
				set_current_pts(dec_ctx->timing, time_to_pes_time(time));
				frames_since_ref_time = 0;
				set_fts(dec_ctx->timing);
			}
			else if (time == WTV_CC_TIMESTAMP_MAGIC && stream_id != alt_stream) {
				use_alt_stream++;
				mprint("WARNING: %i WTV_CC_TIMESTAMP_MAGIC detected in cc timestamps. \n", use_alt_stream);
				if (use_alt_stream == WTV_CC_TIMESTAMP_MAGIC_THRESH)
					mprint("WARNING: WTV_CC_TIMESTAMP_MAGIC_THRESH reached. Switching to alt timestamps!\n");
			}
			len-=16;
		}
		if( !memcmp(guid, WTV_DATA, 16 )
				&& check_stream_id(stream_id, video_streams, num_streams) && dec_ctx->timing->current_pts != 0
				&& (ccx_options.wtvmpeg2 || (!ccx_options.wtvmpeg2 && len==2)))
		{
			// This is the data for a stream we want to process
			dbg_print(CCX_DMT_PARSE, "\nWTV DATA\n");
			get_sized_buffer(ctx->demux_ctx, cb, len);
			if(cb->buffer==NULL)
				return CCX_EOF;
			memcpy(data->buffer+data->len, cb->buffer, len);
			data->len+=result;
			bytesread+=(int) len;
			frames_since_ref_time++;
			set_fts(dec_ctx->timing);
			if(pad>0)
			{ //Make sure we skip any padding too, since we are returning here
				skip_sized_buffer(ctx->demux_ctx, cb, pad);
			}
			return bytesread;
		}
		if(len+pad>0)
		{
			// skip any remaining data
			// For any unhandled GUIDs this will be len+pad
			// For others it will just be pad
			skip_sized_buffer(ctx->demux_ctx, cb, len+pad);
		}
	}
}
Example #5
0
// read_header will read all the required information from
// the wtv header/root sections and calculate the skip_chunks.
// If successful, will return with the file positioned
// at the start of the data dir
int read_header(struct ccx_demuxer *ctx, struct wtv_chunked_buffer *cb)
{
	ctx->startbytes_avail = (int)buffered_read_opt(ctx, ctx->startbytes, STARTBYTESLENGTH);
	return_to_buffer(ctx, ctx->startbytes, ctx->startbytes_avail);

	uint8_t *parsebuf;
	parsebuf = (uint8_t*)malloc(1024);
	buffered_read(ctx, parsebuf,0x42);
	ctx->past+=result;
	if (result!=0x42)
	{
		mprint("\nPremature end of file!\n");
		return CCX_EOF;
	}
	// Expecting WTV header
	if( !memcmp(parsebuf, WTV_HEADER, 16 ) )
	{
		dbg_print(CCX_DMT_PARSE, "\nWTV header\n");
	}
	else
	{
		mprint("\nMissing WTV header. Abort.\n");
		return CCX_EOF;
	}

	//Next read just enough to get the location of the root directory
	uint32_t filelen;
	uint32_t root_dir;
	memcpy(&filelen, parsebuf+0x30, 4);
	dbg_print(CCX_DMT_PARSE, "filelen: %x\n", filelen);
	memcpy(&root_dir, parsebuf+0x38, 4);
	dbg_print(CCX_DMT_PARSE, "root_dir: %x\n", root_dir);

	//Seek to start of the root dir. Typically 0x1100
	buffered_skip(ctx,(root_dir*WTV_CHUNK_SIZE)-0x42);
	ctx->past+=(root_dir*WTV_CHUNK_SIZE)-0x42;

	if (result!=(root_dir*WTV_CHUNK_SIZE)-0x42)
		return CCX_EOF;

	// Read and calculate the meta data chunks in the file we need to skip over
	// while parsing the file.
	int end=0;
	while(!end)
	{
		buffered_read(ctx, parsebuf, 32);
		int x;
		for(x=0; x<16; x++)
			dbg_print(CCX_DMT_PARSE, "%02X ", parsebuf[x]);
		dbg_print(CCX_DMT_PARSE, "\n");

		if (result!=32)
		{
			mprint("\nPremature end of file!\n");
			free(parsebuf);
			return CCX_EOF;
		}
		ctx->past+=32;
		if( !memcmp(parsebuf, WTV_EOF, 16 ))
		{
			dbg_print(CCX_DMT_PARSE, "WTV EOF\n");
			end=1;
		}
		else
		{

			uint16_t len;
			uint64_t file_length;
			memcpy(&len, parsebuf+16, 2);
			dbg_print(CCX_DMT_PARSE, "len: %x\n", len);
			memcpy(&file_length, parsebuf+24, 8);
			dbg_print(CCX_DMT_PARSE, "file_length: %x\n", file_length);
			if(len>1024)
			{
				mprint("Too large for buffer!\n");
				free(parsebuf);
				return CCX_EOF;
			}
			buffered_read(ctx, parsebuf, len-32);
			if (result!=len-32)
			{
				mprint("Premature end of file!\n");
				free(parsebuf);
				return CCX_EOF;
			}
			ctx->past+=len-32;
			// Read a unicode string
			uint32_t text_len;
			memcpy(&text_len, parsebuf, 4); //text_len is number of unicode chars, not bytes.
			dbg_print(CCX_DMT_PARSE, "text_len: %x\n", text_len);
			char *string;
			string = (char*)malloc(text_len+1); // alloc for ascii
			string[text_len]='\0';
			for(x=0; x<text_len; x++)
			{
				memcpy(&string[x], parsebuf+8+(x*2), 1); // unicode converted to ascii
			}
			dbg_print(CCX_DMT_PARSE, "string: %s\n", string);
			// Ignore everything that doesn't contain the text ".entries."
			if(strstr(string, WTV_TABLE_ENTRIES)!=NULL)
			{
				uint32_t value;
				uint32_t flag;
				memcpy(&value, parsebuf+(text_len*2)+8, 4);
				memcpy(&flag, parsebuf+(text_len*2)+4+8, 4);
				dbg_print(CCX_DMT_PARSE, "value: %x\n", value);
				dbg_print(CCX_DMT_PARSE, "flag: %x\n", flag);
				if(!add_skip_chunks(ctx, cb, value, flag))
				{
					mprint("Premature end of file!\n");
					free(parsebuf);
					return CCX_EOF;
				}
			}
			free(string);
		}
	}
	// Our list of skip_chunks is now complete
	// Sort it (not sure if this is needed, but it doesn't hurt).
	qsort(cb->skip_chunks, cb->count, sizeof(uint64_t), qsort_cmpint);
	dbg_print(CCX_DMT_PARSE, "skip_chunks: ");
	int x;
	for(x=0; x<cb->count; x++)
		dbg_print(CCX_DMT_PARSE, "%llx, ", cb->skip_chunks[x]);
	dbg_print(CCX_DMT_PARSE, "\n");

	// Seek forward to the start of the data dir
	// Typically 0x40000
	buffered_skip(ctx,(int)((cb->skip_chunks[cb->chunk]+WTV_META_CHUNK_SIZE)-ctx->past));
	cb->filepos=(cb->skip_chunks[cb->chunk]+WTV_META_CHUNK_SIZE);
	cb->chunk++;
	ctx->past=cb->filepos;
	free(parsebuf);
	return CCX_OK;
}
Example #6
0
// Return 1 for sucessfully read ts packet
int ts_readpacket(struct ccx_demuxer* ctx, struct ts_payload *payload)
{
	unsigned int adaptation_field_length = 0;
	unsigned int adaptation_field_control;
	long long result;
	if (ctx->m2ts)
	{
		/* M2TS just adds 4 bytes to each packet (so size goes from 188 to 192)
		   The 4 bytes are not important to us, so just skip
		// TP_extra_header {   
		Copy_permission_indicator 2  unimsbf
		Arrival_time_stamp 30 unimsbf
		} */
		unsigned char tp_extra_header[4];
		result = buffered_read(ctx, tp_extra_header, 3);
		ctx->past += result;
		if (result != 4)
		{
			if (result>0)
				mprint("Premature end of file!\n");
			return CCX_EOF;
		}
	}

	result = buffered_read(ctx, tspacket, 188);
	ctx->past += result;
	if (result != 188)
	{
		if (result > 0)
			mprint("Premature end of file!\n");
		return CCX_EOF;
	}

	int printtsprob = 1;
	while (tspacket[0]!=0x47)
	{
		if (printtsprob)
		{
			dbg_print(CCX_DMT_DUMPDEF,"\nProblem: No TS header mark (filepos=%lld). Received bytes:\n", ctx->past);
			dump(CCX_DMT_DUMPDEF, tspacket, 4, 0, 0);

			dbg_print(CCX_DMT_DUMPDEF, "Skip forward to the next TS header mark.\n");
			printtsprob = 0;
		}

		unsigned char *tstemp;
		// The amount of bytes read into tspacket
		int tslen = 188;

		// Check for 0x47 in the remaining bytes of tspacket
		tstemp = (unsigned char *) memchr (tspacket+1, 0x47, tslen-1);
		if (tstemp != NULL )
		{
			// Found it
			int atpos = tstemp-tspacket;

			memmove (tspacket,tstemp,(size_t)(tslen-atpos));
			result = buffered_read(ctx, tspacket+(tslen-atpos), atpos);
			ctx->past+=result;
			if (result!=atpos)
			{
				mprint("Premature end of file!\n");
				return CCX_EOF;
			}
		}
		else
		{
			// Read the next 188 bytes.
			result = buffered_read(ctx, tspacket, tslen);
			ctx->past+=result;
			if (result!=tslen)
			{
				mprint("Premature end of file!\n");
				return CCX_EOF;
			}
		}
	}


	payload->transport_error = (tspacket[1]&0x80)>>7;
	payload->pesstart =  (tspacket[1] & 0x40) >> 6;
	// unsigned transport_priority = (tspacket[1]&0x20)>>5;
	payload->pid = (((tspacket[1] & 0x1F) << 8) | tspacket[2]) & 0x1FFF;
	// unsigned transport_scrambling_control = (tspacket[3]&0xC0)>>6;
	adaptation_field_control = (tspacket[3]&0x30)>>4;
	payload->counter = tspacket[3] & 0xF;

	if (payload->transport_error)
	{
		dbg_print(CCX_DMT_DUMPDEF, "Warning: Defective (error indicator on) TS packet (filepos=%lld):\n", ctx->past);
		dump(CCX_DMT_DUMPDEF, tspacket, 188, 0, 0);
	}

	payload->start = tspacket + 4;
	payload->length = 188 - 4;
	if ( adaptation_field_control & 2 )
	{
		// Take the PCR (Program Clock Reference) from here, in case PTS is not available (copied from telxcc).
		adaptation_field_length = tspacket[4];

		uint8_t af_pcr_exists = (tspacket[5] & 0x10) >> 4;
		if (af_pcr_exists > 0 )
		{
			payload->pcr = 0;
			payload->pcr |= (tspacket[6] << 25);
			payload->pcr |= (tspacket[7] << 17);
			payload->pcr |= (tspacket[8] << 9);
			payload->pcr |= (tspacket[9] << 1);
			payload->pcr |= (tspacket[10] >> 7);
			/* Ignore 27 Mhz clock since we dont deal in nano seconds*/
			//payload->pcr = ((tspacket[10] & 0x01) << 8);
			//payload->pcr |= tspacket[11];
		}
Example #7
0
/* Read and evaluate the current video PES header. The function returns
 * the length of the payload if successful, otherwise -1 is returned
 * indicating a premature end of file / too small buffer.
 * If sbuflen is
 *    0 .. Read from file into nextheader
 *    >0 .. Use data in nextheader with the length of sbuflen
 */
int read_video_pes_header (unsigned char *nextheader, int *headerlength, int sbuflen)
{
    // Read the next video PES
    // ((nextheader[3]&0xf0)==0xe0)
    unsigned peslen=nextheader[4]<<8 | nextheader[5];
    unsigned payloadlength = 0; // Length of packet data bytes

    if ( !sbuflen )
    {
        // Extension present, get it
        buffered_read (nextheader+6,3);
        past=past+result;
        if (result!=3) {
            // Consider this the end of the show.
            return -1;
        }
    }
    else
    {
	// We need at least 9 bytes to continue
	if( sbuflen < 9 )
	    return -1;
    }
    *headerlength = 6+3;

    unsigned hskip=0;

    // Assume header[8] is right, but double check
    if ( !sbuflen )
    {
        if (nextheader[8] > 0) {
            buffered_read (nextheader+9,nextheader[8]);
            past=past+result;
            if (result!=nextheader[8]) {
                return -1;
            }
        }
    }
    else
    {
	// See if the buffer is big enough
	if( sbuflen < *headerlength + (int)nextheader[8] )
	    return -1;
    }
    *headerlength += (int) nextheader[8];
    int falsepes = 0;
    int pesext = 0;

    // Avoid false positives, check --- not really needed
    if ( (nextheader[7]&0xC0) == 0x80 ) {
        // PTS only
        hskip += 5;
        if( (nextheader[9]&0xF1) != 0x21 || (nextheader[11]&0x01) != 0x01
            || (nextheader[13]&0x01) != 0x01 ) {
            falsepes = 1;
            printf("False PTS\n");
        }
    } else if ( (nextheader[7]&0xC0) == 0xC0 ) {
        // PTS and DTS
        hskip += 10;
        if( (nextheader[9]&0xF1) != 0x31 || (nextheader[11]&0x01) != 0x01
            || (nextheader[13]&0x01) != 0x01
            || (nextheader[14]&0xF1) != 0x11 || (nextheader[16]&0x01) != 0x01
            || (nextheader[18]&0x01) != 0x01 ) {
            falsepes = 1;
            printf("False PTS/DTS\n");
        }
    } else if ( (nextheader[7]&0xC0) == 0x40 ) {
        // Forbidden
        falsepes = 1;
        printf("False PTS/DTS flag\n");
    }
    if ( !falsepes && nextheader[7]&0x20 ) { // ESCR
        if ((nextheader[9+hskip]&0xC4) != 0x04 || !(nextheader[11+hskip]&0x04)
            || !(nextheader[13+hskip]&0x04) || !(nextheader[14+hskip]&0x01) ) {
            falsepes = 1;
            printf("False ESCR\n");
        }
        hskip += 6;
    }
    if ( !falsepes && nextheader[7]&0x10 ) { // ES
        if ( !(nextheader[9+hskip]&0x80) || !(nextheader[11+hskip]&0x01) ) {
            printf("False ES\n");
            falsepes = 1;
        }
        hskip += 3;
    }
    if ( !falsepes && nextheader[7]&0x04) { // add copy info
        if ( !(nextheader[9+hskip]&0x80) ) {
            printf("False add copy info\n");
            falsepes = 1;
        }
        hskip += 1;
    }
    if ( !falsepes && nextheader[7]&0x02) { // PES CRC
        hskip += 2;
    }
    if ( !falsepes && nextheader[7]&0x01) { // PES extension
        if ( (nextheader[9+hskip]&0x0E)!=0x0E ) {
            printf("False PES ext\n");
            falsepes = 1;
        }
        hskip += 1;
        pesext = 1;
    }

    if ( !falsepes ) {
        hskip = nextheader[8];
    }

    if ( !falsepes && nextheader[7]&0x80 ) {
        // Read PTS from byte 9,10,11,12,13

        unsigned bits_9 =  (nextheader[9] & 0x0E) << 29;
        unsigned bits_10 = nextheader[10] << 22;
        unsigned bits_11 = (nextheader[11] & 0xFE) << 14;
        unsigned bits_12 = nextheader[12] << 7;
        unsigned bits_13 = nextheader[13] >> 1;
        current_pts = bits_9 | bits_10 | bits_11 | bits_12 | bits_13;

        if (pts_set==0)
            pts_set=1;

        if (debug_verbose)
            printf("Set PTS: %s (%u)\n",
                   print_mstime((current_pts)/(MPEG_CLOCK_FREQ/1000)),
                   unsigned(current_pts) );
        /* The user data holding the captions seems to come between GOP and
         * the first frame. The sync PTS (sync_pts) (set at picture 0)
         * corresponds to the first frame of the current GOP. */
    }
Example #8
0
int main(int argc, char *argv[]) {
    int argchar;
	char* progname = argv[0];
	sl* infns = sl_new(16);
	char* outfnpat = NULL;
	char* racol = "RA";
	char* deccol = "DEC";
	char* tempdir = "/tmp";
	anbool gzip = FALSE;
	sl* cols = sl_new(16);
	int loglvl = LOG_MSG;
	int nside = 1;
	double margin = 0.0;
	int NHP;
	double md;
	char* backref = NULL;
	
	fitstable_t* intable;
	fitstable_t** outtables;

	char** myargs;
	int nmyargs;
	int i;

    while ((argchar = getopt (argc, argv, OPTIONS)) != -1)
        switch (argchar) {
		case 'b':
			backref = optarg;
			break;
		case 't':
			tempdir = optarg;
			break;
		case 'c':
			sl_append(cols, optarg);
			break;
		case 'g':
			gzip = TRUE;
			break;
		case 'o':
			outfnpat = optarg;
			break;
		case 'r':
			racol = optarg;
			break;
		case 'd':
			deccol = optarg;
			break;
		case 'n':
			nside = atoi(optarg);
			break;
		case 'm':
			margin = atof(optarg);
			break;
		case 'v':
			loglvl++;
			break;
        case '?':
            fprintf(stderr, "Unknown option `-%c'.\n", optopt);
        case 'h':
			printHelp(progname);
            return 0;
        default:
            return -1;
        }

	if (sl_size(cols) == 0) {
		sl_free2(cols);
		cols = NULL;
	}

	nmyargs = argc - optind;
	myargs = argv + optind;

	for (i=0; i<nmyargs; i++)
		sl_append(infns, myargs[i]);
	
	if (!sl_size(infns)) {
		printHelp(progname);
		printf("Need input filenames!\n");
		exit(-1);
	}
	log_init(loglvl);
	fits_use_error_system();

	NHP = 12 * nside * nside;
	logmsg("%i output healpixes\n", NHP);
	outtables = calloc(NHP, sizeof(fitstable_t*));
	assert(outtables);

	md = deg2dist(margin);

	/**
	 About the mincaps/maxcaps:

	 These have a center and radius-squared, describing the region
	 inside a small circle on the sphere.

	 The "mincaps" describe the regions that are definitely owned by a
	 single healpix -- ie, more than MARGIN distance from any edge.
	 That is, the mincap is the small circle centered at (0.5, 0.5) in
	 the healpix and with radius = the distance to the closest healpix
	 boundary, MINUS the margin distance.

	 Below, we first check whether a new star is within the "mincap"
	 of any healpix.  If so, we stick it in that healpix and continue.

	 Otherwise, we check all the "maxcaps" -- these are the healpixes
	 it could *possibly* be in.  We then refine with
	 healpix_within_range_of_xyz.  The maxcap distance is the distance
	 to the furthest boundary point, PLUS the margin distance.
	 */


	cap_t* mincaps = malloc(NHP * sizeof(cap_t));
	cap_t* maxcaps = malloc(NHP * sizeof(cap_t));
	for (i=0; i<NHP; i++) {
		// center
		double r2;
		double xyz[3];
		double* cxyz;
		double step = 1e-3;
		double v;
		double r2b, r2a;

		cxyz = mincaps[i].xyz;
		healpix_to_xyzarr(i, nside, 0.5, 0.5, mincaps[i].xyz);
		memcpy(maxcaps[i].xyz, cxyz, 3 * sizeof(double));
		logverb("Center of HP %i: (%.3f, %.3f, %.3f)\n", i, cxyz[0], cxyz[1], cxyz[2]);

		// radius-squared:
		// max is the easy one: max of the four corners (I assume)
		r2 = 0.0;
		healpix_to_xyzarr(i, nside, 0.0, 0.0, xyz);
		logverb("  HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
		r2 = MAX(r2, distsq(xyz, cxyz, 3));
		healpix_to_xyzarr(i, nside, 1.0, 0.0, xyz);
		logverb("  HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
		r2 = MAX(r2, distsq(xyz, cxyz, 3));
		healpix_to_xyzarr(i, nside, 0.0, 1.0, xyz);
		logverb("  HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
		r2 = MAX(r2, distsq(xyz, cxyz, 3));
		healpix_to_xyzarr(i, nside, 1.0, 1.0, xyz);
		logverb("  HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
		r2 = MAX(r2, distsq(xyz, cxyz, 3));
		logverb("  max distsq: %.3f\n", r2);
		logverb("  margin dist: %.3f\n", md);
		maxcaps[i].r2 = square(sqrt(r2) + md);
		logverb("  max cap distsq: %.3f\n", maxcaps[i].r2);
		r2a = r2;

		r2 = 1.0;
		r2b = 0.0;
		for (v=0; v<=1.0; v+=step) {
			healpix_to_xyzarr(i, nside, 0.0, v, xyz);
			r2 = MIN(r2, distsq(xyz, cxyz, 3));
			r2b = MAX(r2b, distsq(xyz, cxyz, 3));
			healpix_to_xyzarr(i, nside, 1.0, v, xyz);
			r2 = MIN(r2, distsq(xyz, cxyz, 3));
			r2b = MAX(r2b, distsq(xyz, cxyz, 3));
			healpix_to_xyzarr(i, nside, v, 0.0, xyz);
			r2 = MIN(r2, distsq(xyz, cxyz, 3));
			r2b = MAX(r2b, distsq(xyz, cxyz, 3));
			healpix_to_xyzarr(i, nside, v, 1.0, xyz);
			r2 = MIN(r2, distsq(xyz, cxyz, 3));
			r2b = MAX(r2b, distsq(xyz, cxyz, 3));
		}
		mincaps[i].r2 = square(MAX(0, sqrt(r2) - md));
		logverb("\nhealpix %i: min rad    %g\n", i, sqrt(r2));
		logverb("healpix %i: max rad    %g\n", i, sqrt(r2a));
		logverb("healpix %i: max rad(b) %g\n", i, sqrt(r2b));
		assert(r2a >= r2b);
	}

	if (backref) {
		fitstable_t* tab = fitstable_open_for_writing(backref);
		int maxlen = 0;
		char* buf;
		for (i=0; i<sl_size(infns); i++) {
			char* infn = sl_get(infns, i);
			maxlen = MAX(maxlen, strlen(infn));
		}
		fitstable_add_write_column_array(tab, fitscolumn_char_type(), maxlen,
										 "filename", NULL);
		fitstable_add_write_column(tab, fitscolumn_i16_type(), "index", NULL);
		if (fitstable_write_primary_header(tab) ||
			fitstable_write_header(tab)) {
			ERROR("Failed to write header of backref table \"%s\"", backref);
			exit(-1);
		}
		buf = malloc(maxlen+1);
		assert(buf);

		for (i=0; i<sl_size(infns); i++) {
			char* infn = sl_get(infns, i);
			int16_t ind;
			memset(buf, 0, maxlen);
			strcpy(buf, infn);
			ind = i;
			if (fitstable_write_row(tab, buf, &ind)) {
				ERROR("Failed to write row %i of backref table: %s = %i",
					  i, buf, ind);
				exit(-1);
			}
		}
		if (fitstable_fix_header(tab) ||
			fitstable_close(tab)) {
			ERROR("Failed to fix header & close backref table");
			exit(-1);
		}
		logmsg("Wrote backref table %s\n", backref);
		free(buf);
	}

	for (i=0; i<sl_size(infns); i++) {
		char* infn = sl_get(infns, i);
		char* originfn = infn;
		int r, NR;
		tfits_type any, dubl;
		il* hps = NULL;
		bread_t* rowbuf;
		int R;
		char* tempfn = NULL;
		char* padrowdata = NULL;
		int ii;

		logmsg("Reading input \"%s\"...\n", infn);

		if (gzip) {
			char* cmd;
			int rtn;
			tempfn = create_temp_file("hpsplit", tempdir);
			asprintf_safe(&cmd, "gunzip -cd %s > %s", infn, tempfn);
			logmsg("Running: \"%s\"\n", cmd);
			rtn = run_command_get_outputs(cmd, NULL, NULL);
			if (rtn) {
				ERROR("Failed to run command: \"%s\"", cmd);
				exit(-1);
			}
			free(cmd);
			infn = tempfn;
		}

		intable = fitstable_open(infn);
		if (!intable) {
			ERROR("Couldn't read catalog %s", infn);
			exit(-1);
		}
		NR = fitstable_nrows(intable);
		logmsg("Got %i rows\n", NR);

		any = fitscolumn_any_type();
		dubl = fitscolumn_double_type();

		fitstable_add_read_column_struct(intable, dubl, 1, 0, any, racol, TRUE);
		fitstable_add_read_column_struct(intable, dubl, 1, sizeof(double), any, deccol, TRUE);

		fitstable_use_buffered_reading(intable, 2*sizeof(double), 1000);

		R = fitstable_row_size(intable);
		rowbuf = buffered_read_new(R, 1000, NR, refill_rowbuffer, intable);

		if (fitstable_read_extension(intable, 1)) {
			ERROR("Failed to find RA and DEC columns (called \"%s\" and \"%s\" in the FITS file)", racol, deccol);
			exit(-1);
		}

		for (r=0; r<NR; r++) {
			int hp = -1;
			double ra, dec;
			int j;
			double* rd;
			void* rowdata;
			void* rdata;

			if (r && ((r % 100000) == 0)) {
			  logmsg("Reading row %i of %i\n", r, NR);
			}

			//printf("reading RA,Dec for row %i\n", r);
			rd = fitstable_next_struct(intable);
			ra = rd[0];
			dec = rd[1];

			logverb("row %i: ra,dec %g,%g\n", r, ra, dec);
			if (margin == 0) {
				hp = radecdegtohealpix(ra, dec, nside);
				logverb("  --> healpix %i\n", hp);
			} else {

				double xyz[3];
				anbool gotit = FALSE;
				double d2;
				if (!hps)
					hps = il_new(4);
				radecdeg2xyzarr(ra, dec, xyz);
				for (j=0; j<NHP; j++) {
					d2 = distsq(xyz, mincaps[j].xyz, 3);
					if (d2 <= mincaps[j].r2) {
						logverb("  -> in mincap %i  (dist %g vs %g)\n", j, sqrt(d2), sqrt(mincaps[j].r2));
						il_append(hps, j);
						gotit = TRUE;
						break;
					}
				}
				if (!gotit) {
					for (j=0; j<NHP; j++) {
						d2 = distsq(xyz, maxcaps[j].xyz, 3);
						if (d2 <= maxcaps[j].r2) {
							logverb("  -> in maxcap %i  (dist %g vs %g)\n", j, sqrt(d2), sqrt(maxcaps[j].r2));
							if (healpix_within_range_of_xyz(j, nside, xyz, margin)) {
								logverb("  -> and within range.\n");
								il_append(hps, j);
							}
						}
					}
				}

				//hps = healpix_rangesearch_radec(ra, dec, margin, nside, hps);

				logverb("  --> healpixes: [");
				for (j=0; j<il_size(hps); j++)
					logverb(" %i", il_get(hps, j));
				logverb(" ]\n");
			}

			//printf("Reading rowdata for row %i\n", r);
			rowdata = buffered_read(rowbuf);
			assert(rowdata);


			j=0;
			while (1) {
				if (hps) {
					if (j >= il_size(hps))
						break;
					hp = il_get(hps, j);
					j++;
				}
				assert(hp < NHP);
				assert(hp >= 0);

				if (!outtables[hp]) {
					char* outfn;
					fitstable_t* out;

					// MEMLEAK the output filename.  You'll live.
					asprintf_safe(&outfn, outfnpat, hp);
					logmsg("Opening output file \"%s\"...\n", outfn);
					out = fitstable_open_for_writing(outfn);
					if (!out) {
						ERROR("Failed to open output table \"%s\"", outfn);
						exit(-1);
					}
					// Set the output table structure.
					if (cols) {
					  fitstable_add_fits_columns_as_struct3(intable, out, cols, 0);
					} else
						fitstable_add_fits_columns_as_struct2(intable, out);

					if (backref) {
						tfits_type i16type;
						tfits_type i32type;
						// R = fitstable_row_size(intable);
						int off = R;
						i16type = fitscolumn_i16_type();
						i32type = fitscolumn_i32_type();
						fitstable_add_read_column_struct(out, i16type, 1, off,
														 i16type, "backref_file", TRUE);
						off += sizeof(int16_t);
						fitstable_add_read_column_struct(out, i32type, 1, off,
														 i32type, "backref_index", TRUE);
					}

					//printf("Output table:\n");
					//fitstable_print_columns(out);

					if (fitstable_write_primary_header(out) ||
						fitstable_write_header(out)) {
						ERROR("Failed to write output file headers for \"%s\"", outfn);
						exit(-1);
					}
					outtables[hp] = out;
				}

				if (backref) {
					int16_t brfile;
					int32_t brind;
					if (!padrowdata) {
						padrowdata = malloc(R + sizeof(int16_t) + sizeof(int32_t));
						assert(padrowdata);
					}
					// convert to FITS endian
					brfile = htons(i);
					brind  = htonl(r);
					// add backref data to rowdata
					memcpy(padrowdata, rowdata, R);
					memcpy(padrowdata + R, &brfile, sizeof(int16_t));
					memcpy(padrowdata + R + sizeof(int16_t), &brind, sizeof(int32_t));
					rdata = padrowdata;
				} else {
					rdata = rowdata;
				}

				if (cols) {
				  if (fitstable_write_struct_noflip(outtables[hp], rdata)) {
				    ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp);
				  }
				} else {
				  if (fitstable_write_row_data(outtables[hp], rdata)) {
				    ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp);
				  }
				}

				if (!hps)
					break;
			}
			if (hps)
				il_remove_all(hps);

		}
		buffered_read_free(rowbuf);
		// wack... buffered_read_free() just frees its internal buffer,
		// not the "rowbuf" struct itself.
		// who wrote this crazy code?  Oh, me of 5 years ago.  Jerk.
		free(rowbuf);

		fitstable_close(intable);
		il_free(hps);

		if (tempfn) {
			logverb("Removing temp file %s\n", tempfn);
			if (unlink(tempfn)) {
				SYSERROR("Failed to unlink() temp file \"%s\"", tempfn);
			}
			tempfn = NULL;
		}

		// fix headers so that the files are valid at this point.
		for (ii=0; ii<NHP; ii++) {
		  if (!outtables[ii])
		    continue;
		  off_t offset = ftello(outtables[ii]->fid);
		  if (fitstable_fix_header(outtables[ii])) {
		    ERROR("Failed to fix header for healpix %i after reading input file \"%s\"", ii, originfn);
		    exit(-1);
		  }
		  fseeko(outtables[ii]->fid, offset, SEEK_SET);
		}

		if (padrowdata) {
			free(padrowdata);
			padrowdata = NULL;
		}

	}

	for (i=0; i<NHP; i++) {
		if (!outtables[i])
			continue;
		if (fitstable_fix_header(outtables[i]) ||
			fitstable_fix_primary_header(outtables[i]) ||
			fitstable_close(outtables[i])) {
			ERROR("Failed to close output table for healpix %i", i);
			exit(-1);
		}
	}

	free(outtables);
	sl_free2(infns);
	sl_free2(cols);

	free(mincaps);
	free(maxcaps);

    return 0;
}
Example #9
0
void* fitstable_next_struct(fitstable_t* tab) {
    if (!tab->br) return NULL;
    return buffered_read(tab->br);
}
Example #10
0
// Return 1 for sucessfully read ts packet
int ts_readpacket(void)
{
    buffered_read(tspacket,188);
    past+=result;
    if (result!=188)
    {
        if (result>0)
            mprint("Premature end of file!\n");
        end_of_file=1;
        return 0;
    }

    int printtsprob = 1;
    while (tspacket[0]!=0x47)
    {
        if (printtsprob)
        {
            mprint ("\nProblem: No TS header mark (filepos=%lld). Received bytes:\n", past);
            dump (DMT_GENERIC_NOTICES, tspacket,4, 0, 0);

            mprint ("Skip forward to the next TS header mark.\n");
            printtsprob = 0;
        }

        unsigned char *tstemp;
        // The amount of bytes read into tspacket
        int tslen = 188;

        // Check for 0x47 in the remaining bytes of tspacket
        tstemp = (unsigned char *) memchr (tspacket+1, 0x47, tslen-1);
        if (tstemp != NULL )
        {
            // Found it
            int atpos = tstemp-tspacket;

            memmove (tspacket,tstemp,(size_t)(tslen-atpos));
            buffered_read(tspacket+(tslen-atpos),atpos);
            past+=result;
            if (result!=atpos) 
            {
                mprint("Premature end of file!\n");
                end_of_file=1;
                return 0;
            }
        }
        else
        {
            // Read the next 188 bytes.
            buffered_read(tspacket,tslen);
            past+=result;
            if (result!=tslen) 
            {
                mprint("Premature end of file!\n");
                end_of_file=1;
                return 0;
            }
        }
    }

    unsigned char *payload_start = tspacket + 4;
    unsigned payload_length = 188 - 4;

    unsigned transport_error_indicator = (tspacket[1]&0x80)>>7;
    unsigned payload_start_indicator = (tspacket[1]&0x40)>>6;
    // unsigned transport_priority = (tspacket[1]&0x20)>>5;
    unsigned pid = (((tspacket[1] & 0x1F) << 8) | tspacket[2]) & 0x1FFF;
    // unsigned transport_scrambling_control = (tspacket[3]&0xC0)>>6;
    unsigned adaptation_field_control = (tspacket[3]&0x30)>>4;
    unsigned ccounter = tspacket[3] & 0xF;

    if (transport_error_indicator)
    {
        mprint ("Warning: Defective (error indicator on) TS packet (filepos=%lld):\n", past);
        dump (DMT_GENERIC_NOTICES, tspacket, 188, 0, 0);
    }

    unsigned adaptation_field_length = 0;
    if ( adaptation_field_control & 2 )
    {
		// Take the PCR (Program Clock Reference) from here, in case PTS is not available (copied from telxcc).
        adaptation_field_length = tspacket[4];

		uint8_t af_pcr_exists = (tspacket[5] & 0x10) >> 4;
			if (af_pcr_exists > 0) {
				uint64_t pts = 0;
				pts |= (tspacket[6] << 25);
				pts |= (tspacket[7] << 17);
				pts |= (tspacket[8] << 9);
				pts |= (tspacket[9] << 1);
				pts |= (tspacket[10] >> 7);
				global_timestamp = (uint32_t) pts / 90;
				pts = ((tspacket[10] & 0x01) << 8);
				pts |= tspacket[11];
				global_timestamp += (uint32_t) (pts / 27000);
				if (!global_timestamp_inited)
				{
					min_global_timestamp = global_timestamp;
					global_timestamp_inited = 1;
				}
			}
Example #11
0
int
main(int argc, char *argv[])
{
	char *load_file = (char *)NULL;	/* DB load path/file name */
	char *optstring;		/* Command option string */
	char *logfile = NULL;	/* Log file name for samfsrestore -g option */
	struct sam_stat sb;
	int option;		/* Current getopt option being processed */
	int	io_sz, read_size;
	int len;
	operator_t	operator;	/* Operator data */
	shm_alloc_t	master_shm;	/* Master shared memory structure */

	program_name = basename(argv[0]);

	operation = NONE;
	replace_newer = false;
	replace = false;
	swapped = false;
	verbose = false;
	quiet = false;
	noheaders = false;
	strip_slashes = false;
	list_by_inode = false;
	online_data = false;
	partial_data = false;
	scan_only = false;
	unarchived_data = false;
	use_file_list = false;
	read_buffer_size = write_buffer_size = CSD_DEFAULT_BUFSZ;
	block_size = 0;
	memset(&csd_header, '\0', sizeof (csd_header));

	CustmsgInit(0, NULL);
	if ((Initial_path = getcwd(NULL, (MAXPATHLEN + 1))) == NULL) {
		error(1, errno, catgets(catfd, SET, 590, "cannot getcwd()"));
	}

	/*
	 *	If basename is samfsrestore, fake out "r" option; if
	 *	samfsdump, fake out "c" option.  Allow redundant specification
	 *	specifications like "samfsrestore r", "samfsdump c".	Allow
	 *	"samfsrestore t" to override "r" option so that we look
	 *	like "ufsrestore t".	If basename is none of the above, then
	 *	user must specify one of c,r,t,O.
	 */

	if (strcmp(program_name, "samfsrestore") == 0) {
		operation = RESTORE;
		optstring = RESTORE_OPT;
		qfs = false;
	} else if (strcmp(program_name, "samfsdump") == 0) {
		operation = DUMP;
		optstring = DUMP_OPT;
		qfs = false;
	} else if (strcmp(program_name, "qfsdump") == 0) {
		operation = DUMP;
		optstring = QFSDUMP_OPT;
		qfs = true;
	} else if (strcmp(program_name, "qfsrestore") == 0) {
		operation = RESTORE;
		optstring = RESTORE_OPT;
		qfs = true;
	}
	if (operation == NONE) {
		error(1, 0, catgets(catfd, SET, 13508,
		    "Improper invocation of SAM-FS csd "
		    "utility %s"),
		    program_name);
	}

	for (nexcluded = CSD_MAX_EXCLUDED - 1; nexcluded >= 0; nexcluded--) {
		excluded[nexcluded] = NULL;
	}
	nexcluded = 0;
	for (nincluded = CSD_MAX_INCLUDED - 1; nincluded >= 0; nincluded--) {
		included[nincluded] = NULL;
	}
	nincluded = 0;

	while ((option = getopt(argc, argv, optstring)) != EOF) {
		switch (option) {
		case 'B':		/* select buffer size */
			io_sz = atoi(optarg) * 512;
			if (io_sz < CSD_MIN_BUFSZ || io_sz > CSD_MAX_BUFSZ) {
				error(1, 0,
				    catgets(catfd, SET, 13509,
				    "Invalid buffer size, must "
				    "be >= %d and <= %d, "
				    "specified in 512b units"),
				    CSD_MIN_BUFSZ/512, CSD_MAX_BUFSZ/512);
			}
			read_buffer_size = write_buffer_size = io_sz;
			break;

		case 'b':		/* select blocking factor */
			io_sz = atoi(optarg) * 512;
			if (io_sz < 0 || io_sz > CSD_MAX_BUFSZ) {
				error(1, 0, catgets(catfd, SET, 13510,
				    "Invalid blocking factor, "
				    "must be >= 0 and <= %d, "
				    "specified in 512b units"),
				    CSD_MAX_BUFSZ/512);
			}
			block_size = io_sz;
			break;

		case 'd':		/* enable debug messages */
			debugging = 1;
			break;

		case 'D':		/* enable directio */
			Directio++;
			break;

		case 'I':		/* file of paths to process */
			if (nincluded > CSD_MAX_INCLUDED) {
				error(1, 0, catgets(catfd, SET, 13537,
		"Maximum -I (include file) entries exceeded, max. is %d."),
				    CSD_MAX_INCLUDED);
			}
			included[nincluded] = optarg;
			len = strlen(optarg);
			if (len > 0) {
				*(included[nincluded] + len) = '\0';
				nincluded++;
			} else {
				error(1, 0, catgets(catfd, SET, 13538,
				    "-I (include file) entry "
				    "<%s> invalid"),
				    optarg);
			}
			break;

		case 'X':		/* excluded directory list */
			if (nexcluded > CSD_MAX_EXCLUDED) {
				error(1, 0, catgets(catfd, SET, 13520,
				    "Maximum -X (excluded "
				    "directory) entries "
				    "exceeded, max. is %d."),
				    CSD_MAX_EXCLUDED);
			}
			excluded[nexcluded] = optarg;
			len = strlen(optarg);
			if (len > 0) {
				*(excluded[nexcluded] + len) = '\0';
				nexcluded++;
			} else {
				error(1, 0, catgets(catfd, SET, 13521,
				    "-X (excluded directory) entry <%s> "
				    "invalid"),
				    optarg);
			}
			break;

		case 'f':		/* specify dump file */
			dump_file = optarg;
			break;

		case 'g':		/* specify log file */
			logfile = optarg;
			break;

		case 'H':		/* no headers */
			noheaders = true;
			break;

		case 'i':		/* print inode numbers */
			ls_options |= LS_INODES;
			break;

		case 'l':		/* print 1 line ls */
			verbose = true;
			ls_options |= LS_LINE1;
			break;

		case 'n':		/* Deprecated. Do not break script. */
			break;

		case 'P':		/* dump partial online data */
			partial_data = true;
			break;

		case 'q':		/* Do not report damage warnings */
			quiet = true;
			break;

		case 'r':		/* replace file if dump is newer */
			replace_newer = true;
			break;

		case 'R':		/* replace file */
			replace = true;
			break;

		case 's':		/* strip leading /'s */
			strip_slashes = true;
			break;

		case 'S':		/* scan only */
			scan_only = true;
			break;

		case 't':		/* list dump file */
			if (operation & DUMP) {
				error(1, 0,
				    catgets(catfd, SET, 941,
				    "Do not specify 't' option."));
			}
			/* samfsrestore t is LIST, not RESTORE */
			operation = LIST;
			break;

		case 'T':		/* statistics mode */
			statistics = true;
			break;

		case 'u':		/* dump unarchived data */
			unarchived_data = true;
			break;

		case 'U':		/* dump online data */
			online_data = true;
			break;

		case 'v':		/* verbose mode */
			verbose = true;
			break;

		case 'W':
			/* warning mode, allowed for historical reasons */
			break;

		case '2':		/* print 2 line ls */
			verbose = true;
			ls_options |= LS_LINE2;
			break;

		case 'Y':		/* file list for dump */
			use_file_list = true;
			break;

		case 'Z':		/* generate database load file	*/
			load_file = optarg;
			break;

		case '?':
			usage();	/* doesn't return */
			break;
		}
	}

	if (scan_only) {		/* clear dump related flags */
		online_data = partial_data = unarchived_data = false;
	}

	if (block_size && (block_size > read_buffer_size)) {
		block_size = read_buffer_size;
	}

	if (operation == DUMP && use_file_list) {
		operation = LISTDUMP;
	}

	if (debugging) {
		int i;

		fprintf(stderr, "\nCOMMAND ARGUMENT VALUES\n");
		fprintf(stderr, "operation = 0x%x\n", operation);
		fprintf(stderr, "debugging = %d\n", debugging);
		fprintf(stderr, "dump_file = %s\n",
		    ((dump_file != (char *)NULL) ? dump_file : "UNDEFINED"));
		fprintf(stderr, "logfile = %s\n",
		    ((logfile != (char *)NULL) ? logfile : "UNDEFINED"));
		fprintf(stderr, "noheaders = %d\n", noheaders);
		fprintf(stderr, "verbose = %d\n", verbose);
		fprintf(stderr, "quiet = %d\n", quiet);
		fprintf(stderr, "qfs = %d\n", qfs);
		fprintf(stderr, "replace = %d\n", replace);
		fprintf(stderr, "replace_newer = %d\n", replace_newer);
		fprintf(stderr, "strip_slashes = %d\n", strip_slashes);
		fprintf(stderr, "ls_options = 0x%x\n", ls_options);
		fprintf(stderr, "load_file = %s\n",
		    ((load_file != (char *)NULL) ? load_file : "UNDEFINED"));
		fprintf(stderr, "online_data = %d\n", online_data);
		fprintf(stderr, "partial_data = %d\n", partial_data);
		fprintf(stderr, "scan_only = %d\n", scan_only);
		fprintf(stderr, "unarchived_data = %d\n", unarchived_data);
		fprintf(stderr, "use_file_list = %d\n", use_file_list);
		fprintf(stderr, "read buffer size = %ld\n", read_buffer_size);
		fprintf(stderr, "write buffer size = %ld\n",
		    write_buffer_size);
		fprintf(stderr, "blocking factor = %ld\n", block_size);

		for (i = optind; i < argc; i++) {
			fprintf(stderr, "File argument %d = %s\n",
			    (i - optind + 1), argv[i]);
		}

		fprintf(stderr, "END COMMAND ARGUMENT VALUES\n\n");
	}

	/*
	 *	If dump file has not been specified, exit with error and usage
	 */

	if ((!scan_only) && dump_file == (char *)NULL) {
		error(0, 0,
		    catgets(catfd, SET, 5019,
		    "%s: Dump file name not specified (-f)"),
		    program_name);
		usage();
	}

	/*
	 *	Access master shared memory segment
	 */

	if ((master_shm.shared_memory = (shm_ptr_tbl_t *)
	    sam_mastershm_attach(0, SHM_RDONLY)) == (void *)-1) {
		/*
		 * If shared memory not available, effective ID must be
		 * super-user.
		 */

		if (geteuid() != 0) {
			error(1, 0,
			    catgets(catfd, SET, 100,
			    "%s may be run only by super-user.\n"),
			    program_name);
		}
	} else {
		/*
		 * If operator is not super-user, issue error message and exit
		 */

		SET_SAM_OPER_LEVEL(operator);

		shmdt((char *)master_shm.shared_memory);

		if (!SAM_ROOT_LEVEL(operator)) {
			error(1, 0,
			    catgets(catfd, SET, 100,
			    "%s may be run only by super-user.\n"),
			    program_name);
		}
	}

	/*
	 *	Open dump file and process header
	 */
	switch (operation) {
	case NONE:
		error(1, 0,
		    catgets(catfd, SET, 1845,
		    "One of c, r, O or t options must be specified."));
		usage();  /* doesn't return */
		break;

	case DUMP:
	case LISTDUMP:
		if (!scan_only) {
			if (strcmp(dump_file, "-") == 0) {
				CSD_fd = dup(STDOUT);
			} else {
				CSD_fd = open(dump_file,
				    O_WRONLY|O_TRUNC|O_CREAT|SAM_O_LARGEFILE,
				    0666 & ~umask(0));
			}

			close(STDOUT);

			if (CSD_fd < 0) {
				error(1, errno,
				    catgets(catfd, SET, 216,
				    "%s: Cannot create dump file"),
				    dump_file);
			}
		}

		break;

	case RESTORE:
	case LIST:
		if (0 == strcmp(dump_file, "-")) {
			CSD_fd = dup(STDIN);
		} else {
			CSD_fd = open(dump_file, O_RDONLY | SAM_O_LARGEFILE);
		}

		close(STDIN);

		if (CSD_fd < 0) {
			error(1, errno,
			    catgets(catfd, SET, 225,
			    "%s: Cannot open dump file"),
			    dump_file);
		}
		if (noheaders) {
			break;
		}
		if (buffered_read(CSD_fd, &csd_header, sizeof (csd_hdr_t)) !=
		    sizeof (csd_hdr_t)) {
			error(1, errno,
			    catgets(catfd, SET, 244,
			    "%s: Header record read error"),
			    dump_file);
		}
		if (csd_header.csd_header.magic != CSD_MAGIC) {
			if (csd_header.csd_header.magic != CSD_MAGIC_RE) {
				error(1, 0,
				    catgets(catfd, SET, 13529,
				    "%s: Volume is not in dump format."),
				    dump_file);
			} else {
				error(0, 0,
				    "Dump file was generated on an "
				    "opposite endian machine");
				swapped = true;
				sam_byte_swap(csd_header_swap_descriptor,
				    &csd_header, sizeof (csd_hdr_t));
			}
		}
		csd_version = csd_header.csd_header.version;
		switch (csd_version) {

		case CSD_VERS_6:
		case CSD_VERS_5:
			/* read remainder of extended header */
			io_sz = sizeof (csd_hdrx_t) - sizeof (csd_hdr_t);
			read_size = buffered_read(CSD_fd,
			    (char *)((char *)(&csd_header) +
			    sizeof (csd_hdr_t)), io_sz);
			if (swapped) {
				sam_bswap4(&csd_header.csd_header_flags, 1);
				sam_bswap4(&csd_header.csd_header_magic, 1);
				sam_bswap8(&csd_header.csd_fs_magic, 1);
			}
			if ((read_size != io_sz) ||
			    csd_header.csd_header_magic != CSD_MAGIC) {
				error(1, errno,
				    catgets(catfd, SET, 244,
				    "%s: Header record read error"),
				    dump_file);
			}
			break;

		case CSD_VERS_4:
		case CSD_VERS_3:
		case CSD_VERS_2:
			break;

		default:
			error(1, 0,
			    catgets(catfd, SET, 5013,
			    "%s: Header record error: Bad version number (%d)"),
			    dump_file, csd_version);
		}

		if (verbose) {
			printf(catgets(catfd, SET, 972, "Dump created:%s"),
			    ctime((const time_t *)&csd_header.csd_header.time));
		}
		if (debugging) {
			fprintf(stderr, "Dump created:%s\n",
			    ctime((const time_t *)&csd_header.csd_header.time));
		}
	}

	if ((logfile != NULL) && (log_st = fopen(logfile, "w")) == NULL) {
		error(0, errno, "%s", logfile);
		error(0, 0, catgets(catfd, SET, 1856,
		    "Open failed on (%s)"), logfile);
	}

	if ((load_file != NULL)) {
		if (0 == strcmp(load_file, "-")) {
			DB_FILE = fdopen(dup(STDOUT), "w");
			close(STDOUT);
		} else if ((DB_FILE = fopen64(load_file, "w")) == NULL) {
			error(0, errno, "%s", load_file);
			error(1, 0, catgets(catfd, SET, 1856,
			    "Open failed on (%s)"), load_file);
		}
	}

	switch (operation) {
	case NONE:
		break;

	case DUMP: {
		char *filename;

		if (debugging) {
			fprintf(stderr, "dumping.  argc %d, initial "
			    "path '%s' \n",
			    optind, Initial_path);
		}
		while ((filename = get_path(argc, argv)) != NULL) {
			if (debugging) {
				fprintf(stderr, "dumping path '%s' \n",
				    filename);
			}
			/*
			 * Try to make sure we don't dump from a non-SAM-FS
			 * filesystem by calling sam_stat() and checking the
			 * attributes.
			 */
			if (sam_stat(filename, &sb, sizeof (sb)) < 0 ||
			    ! SS_ISSAMFS(sb.attr)) {
				BUMP_STAT(errors);
				BUMP_STAT(errors_dir);
				error(1, errno,
				    catgets(catfd, SET, 259,
				    "%s: Not a SAM-FS file."),
				    filename);
			} else {
				if (SAM_fd != -1) {
					close(SAM_fd);
				}
				SAM_fd = open_samfs(filename);
				csd_dump_path("initial", filename,
				    (mode_t)sb.st_mode);
				if (S_ISDIR(sb.st_mode)) {
					process_saved_dir_list(filename);
				}
			}

			if (chdir(Initial_path) < 0) {
				error(1, errno,
				    catgets(catfd, SET, 212,
				    "%s: cannot chdir()"),
				    Initial_path);
			}

		}
		}
		bflush(CSD_fd);
		break;

	case LISTDUMP: {
		char *filename;

		if (debugging) {
			fprintf(stderr, "dumping.  argc %d, initial "
			    "path '%s' \n",
			    optind, Initial_path);
		}

		while ((filename = get_path(argc, argv)) != NULL) {
			if (debugging) {
				fprintf(stderr, "dumping path '%s' \n",
				    filename);
			}
			if (strcmp(filename, "-") == 0) {
				DL_FILE = fdopen(dup(STDIN), "r");
				close(STDIN);
			} else if ((DL_FILE = fopen(filename, "r")) == NULL) {
				error(0, errno, "%s", load_file);
				error(1, 0, catgets(catfd, SET, 1856,
				    "Open failed on (%s)"), load_file);
			}

			if (SAM_fd != -1) {
				close(SAM_fd);
			}
			csd_dump_files(DL_FILE, filename);
			fclose(DL_FILE);
			DL_FILE = NULL;
		}
		}
		bflush(CSD_fd);
		break;

	case RESTORE:
		umask(0);
		cs_restore(strip_slashes, (argc - optind), &argv[optind]);
		break;

	case LIST:
		cs_list((argc - optind), &argv[optind]);
	}

	close(CSD_fd);
	if (statistics) {
		print_stats();
	}
	return (exit_status);
}
Example #12
0
int replay_log_record(HttpHandlerContext * const context,
                      BufferedReadContext * const brc)
{
    DBLog * const db_log = &app_context.db_log;
    
    if (db_log->db_log_fd == -1) {
        return 1;
    }
    char buf_cookie_head[sizeof DB_LOG_RECORD_COOKIE_HEAD - (size_t) 1U];
    char buf_cookie_tail[sizeof DB_LOG_RECORD_COOKIE_TAIL - (size_t) 1U];
    char buf_number[50];
    char *pnt;
    char *endptr;
    const off_t current_offset = lseek(db_log->db_log_fd, (off_t) 0, SEEK_CUR);
    
    ssize_t readnb = buffered_read(brc, buf_cookie_head,
                                   sizeof buf_cookie_head);
    if (readnb == (ssize_t) 0) {
        return 1;
    }
    if (readnb != (ssize_t) sizeof buf_cookie_head ||
        memcmp(buf_cookie_head, DB_LOG_RECORD_COOKIE_HEAD, readnb) != 0) {
        if (lseek(db_log->db_log_fd, current_offset, SEEK_SET) == (off_t) -1 ||
            ftruncate(db_log->db_log_fd, current_offset) != 0) {
        }
        return -1;
    }
    pnt = buf_number;
    for (;;) {
        if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U) {
            return -1;
        }
        if (*pnt == ' ') {
            break;
        }
        if (++pnt == &buf_number[sizeof buf_number]) {
            return -1;
        }
    }
    if (*buf_number == *DB_LOG_RECORD_COOKIE_MARK_CHAR) {
        time_t ts = (time_t) strtol(buf_number + 1U, &endptr, 16);
        if (endptr == NULL || endptr == buf_number + 1U || ts <= (time_t) 0) {
            return -1;
        }
        char t;
        if (buffered_read(brc, &t, (size_t) 1U) != (size_t) 1U ||
            t != *DB_LOG_RECORD_COOKIE_TIMESTAMP_CHAR) {
            return -1;
        }
        readnb = buffered_read(brc, buf_cookie_tail, sizeof buf_cookie_tail);
        if (readnb != (ssize_t) sizeof buf_cookie_tail ||
            memcmp(buf_cookie_tail, DB_LOG_RECORD_COOKIE_TAIL, readnb) != 0) {
            return -1;
        }
        return 0;
    }
    int verb = (int) strtol(buf_number, &endptr, 16);
    if (endptr == NULL || endptr == buf_number) {
        return -1;
    }
    
    pnt = buf_number;
    for (;;) {
        if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U) {
            return -1;
        }
        if (*pnt == ':') {
            break;
        }
        if (++pnt == &buf_number[sizeof buf_number]) {
            return -1;
        }
    }
    size_t uri_len = (size_t) strtoul(buf_number, &endptr, 16);
    if (endptr == NULL || endptr == buf_number || uri_len <= (size_t) 0U) {
        return -1;
    }
    char *uri;
    if ((uri = malloc(uri_len + (size_t) 1U)) == NULL) {
        _exit(1);
    }    
    if (buffered_read(brc, uri, uri_len) != (ssize_t) uri_len) {
        return -1;
    }
    *(uri + uri_len) = 0;
    
    pnt = buf_number;
    if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U ||
        *pnt != ' ') {
        free(uri);
        return -1;
    }    
    for (;;) {
        if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U) {
            free(uri);
            return -1;
        }
        if (*pnt == ':') {
            break;
        }
        if (++pnt == &buf_number[sizeof buf_number]) {
            free(uri);
            return -1;
        }
    }
    size_t body_len = (size_t) strtoul(buf_number, &endptr, 16);
    if (endptr == NULL || endptr == buf_number) {
        free(uri);
        return -1;
    }
    
    char *body = NULL;
    if (body_len > (size_t) 0U) {
        body = malloc(body_len + (size_t) 1U);
        if (body == NULL) {
            free(uri);
            _exit(1);
        }
    }
    if (body != NULL) {
        assert(body_len > (size_t) 0U);
        if (buffered_read(brc, body, body_len) != (ssize_t) body_len) {
            free(uri);
            free(body);
            return -1;
        }
        *(body + body_len) = 0;
    }
    readnb = buffered_read(brc, buf_cookie_tail, sizeof buf_cookie_tail);
    if (readnb != (ssize_t) sizeof buf_cookie_tail ||
        memcmp(buf_cookie_tail, DB_LOG_RECORD_COOKIE_TAIL, readnb) != 0) {
        free(body);
        free(uri);
        return -1;
    }
    fake_request(context, verb, uri, body, body_len, 1, 0);
    free(body);
    free(uri);
    
    return 0;
}
Example #13
0
/**
 * Read from a socket.  (Use this instead of read() because it has
 * buffering.)
 * @param s Socket
 * @param buf Buffer to get
 * @param len Length
 * @return int
 */
int sread(ano_socket_t s, char *buf, int len)
{
    return buffered_read(s, buf, len);
}
Example #14
0
// Return 1 for sucessfully read ts packet
int ts_readpacket(void)
{
    buffered_read(tspacket,188);
    past+=result;
    if (result!=188)
    {
        if (result>0)
            mprint("Premature end of file!\n");
        end_of_file=1;
        return 0;
    }

    int printtsprob = 1;
    while (tspacket[0]!=0x47)
    {
        if (printtsprob)
        {
            mprint ("\nProblem: No TS header mark. Received bytes:\n");
            dump (DMT_GENERIC_NOTICES, tspacket,4, 0, 0);

            mprint ("Skip forward to the next TS header mark.\n");
            printtsprob = 0;
        }

        unsigned char *tstemp;
        // The amount of bytes read into tspacket
        int tslen = 188;

        // Check for 0x47 in the remaining bytes of tspacket
        tstemp = (unsigned char *) memchr (tspacket+1, 0x47, tslen-1);
        if (tstemp != NULL )
        {
            // Found it
            int atpos = tstemp-tspacket;

            memmove (tspacket,tstemp,(size_t)(tslen-atpos));
            buffered_read(tspacket+(tslen-atpos),atpos);
            past+=result;
            if (result!=atpos) 
            {
                mprint("Premature end of file!\n");
                end_of_file=1;
                return 0;
            }
        }
        else
        {
            // Read the next 188 bytes.
            buffered_read(tspacket,tslen);
            past+=result;
            if (result!=tslen) 
            {
                mprint("Premature end of file!\n");
                end_of_file=1;
                return 0;
            }
        }
    }

    unsigned char *payload_start = tspacket + 4;
    unsigned payload_length = 188 - 4;

    unsigned transport_error_indicator = (tspacket[1]&0x80)>>7;
    unsigned payload_start_indicator = (tspacket[1]&0x40)>>6;
    // unsigned transport_priority = (tspacket[1]&0x20)>>5;
    unsigned pid = (((tspacket[1] & 0x1F) << 8) | tspacket[2]) & 0x1FFF;
    // unsigned transport_scrambling_control = (tspacket[3]&0xC0)>>6;
    unsigned adaptation_field_control = (tspacket[3]&0x30)>>4;
    unsigned ccounter = tspacket[3] & 0xF;

    if (transport_error_indicator)
    {
        mprint ("Warning: Defective (error indicator on) TS packet:\n");
        dump (DMT_GENERIC_NOTICES, tspacket, 188, 0, 0);
    }

    unsigned adaptation_field_length = 0;
    if ( adaptation_field_control & 2 )
    {
		// TODO: If present, we should take the PCR (Program Clock Reference) from here, in case PTS is not
		// available, as done in telxcc.
        adaptation_field_length = tspacket[4];

        payload_start = payload_start + adaptation_field_length + 1;
        payload_length = tspacket+188-payload_start;
    }

    dbg_print(DMT_PARSE, "TS pid: %d  PES start: %d  counter: %u  payload length: %u  adapt length: %d\n",
            pid, payload_start_indicator, ccounter, payload_length,
            int(adaptation_field_length));

    // Catch bad packages with adaptation_field_length > 184 and
    // the unsigned nature of payload_length leading to huge numbers.
    if (payload_length > 184)
    {
        // This renders the package invalid
        payload_length = 0;
        dbg_print(DMT_PARSE, "  Reject package - set length to zero.\n");
    }

    // Save data in global struct
    payload.start = payload_start;
    payload.length = payload_length;
    payload.pesstart = payload_start_indicator;
    payload.pid = pid;
    payload.counter = ccounter;
	payload.transport_error = transport_error_indicator;
    if (payload_length == 0)
    {
        dbg_print(DMT_PARSE, "  No payload in package.\n");
    }

    // Store packet information
    return 1;
}