void
readcheck(
void *buffer,
size_t size,
int msgNum)
{
size_t ngot;
if (((ngot = buffered_read(CSD_fd, buffer, size)) != size) &&
ngot != (size_t)-1) {
char *explanation;
explanation = catgets(catfd, SET, msgNum, "??");
error(1, errno,
catgets(catfd, SET, 13502,
"Corrupt samfsdump file. read(%d) returned %d bytes, %s"),
size, ngot, explanation);
} else if (ngot == (size_t)-1) {
char *explanation;
explanation = catgets(catfd, SET, msgNum, "??");
error(1, errno,
catgets(catfd, SET, 13503,
"System error return from read of samfsdump file, %s"),
explanation);
}
}
// get_sized_buffer will alloc and set a buffer in the passed wtv_chunked_buffer struct
// it will handle any meta data chunks that need to be skipped in the file
// Will print error messages and return a null buffer on error.
void get_sized_buffer(struct ccx_demuxer *ctx, struct wtv_chunked_buffer *cb, uint32_t size)
{
if(cb->buffer != NULL && cb->buffer_size > 0)
{
free(cb->buffer);
}
if(size > WTV_MAX_ALLOC)
{
mprint("\nRequested buffer of %i > %i bytes (WTV_MAX_ALLOC)!\n", size, WTV_MAX_ALLOC);
cb->buffer = NULL;
return;
}
cb->buffer = (uint8_t*)malloc(size);
cb->buffer_size = size;
uint64_t start = cb->filepos;
if(cb->skip_chunks[cb->chunk] != -1 && (start + size) > cb->skip_chunks[cb->chunk])
{
buffered_read(ctx, cb->buffer, (int)(cb->skip_chunks[cb->chunk]-start));
cb->filepos += cb->skip_chunks[cb->chunk]-start;
buffered_seek(ctx, WTV_META_CHUNK_SIZE);
cb->filepos += WTV_META_CHUNK_SIZE;
buffered_read(ctx, cb->buffer+(cb->skip_chunks[cb->chunk]-start), (int)(size-(cb->skip_chunks[cb->chunk]-start)));
cb->filepos += size-(cb->skip_chunks[cb->chunk]-start);
cb->chunk++;
}
else
{
buffered_read(ctx, cb->buffer, size);
cb->filepos += size;
if(result != size)
{
free(cb->buffer);
cb->buffer_size = 0;
ctx->past = cb->filepos;
cb->buffer = NULL;
mprint("\nPremature end of file!\n");
return;
}
}
ctx->past = cb->filepos;
return;
}
// Read the actual values of the passed lookup offset and add them to
// the list of chunks to skip as nessasary. Returns false on error.
int add_skip_chunks(struct ccx_demuxer *ctx, struct wtv_chunked_buffer *cb, uint32_t offset, uint32_t flag)
{
uint64_t start = ctx->past;
buffered_seek(ctx, (int)((offset*WTV_CHUNK_SIZE) - start));
uint64_t seek_back=0-((offset*WTV_CHUNK_SIZE)-start);
uint32_t value;
buffered_read(ctx, (unsigned char*)&value, 4);
if(result!=4)
return 0;
seek_back-=4;
while(value!=0)
{
dbg_print(CCX_DMT_PARSE, "value: %llx\n", get_meta_chunk_start(value));
buffered_read(ctx, (unsigned char*)&value, 4);
if(result!=4)
return 0;
add_chunk(cb, get_meta_chunk_start(value));
seek_back-=4;
}
buffered_seek(ctx, (int)seek_back);
return 1;
}
LLONG get_data(struct lib_ccx_ctx *ctx, struct wtv_chunked_buffer *cb, struct demuxer_data *data)
{
static int video_streams[32];
static int alt_stream; //Stream to use for timestamps if the cc stream has broken timestamps
static int use_alt_stream = 0;
static int num_streams = 0;
struct lib_cc_decode *dec_ctx = update_decoder_list(ctx);
while(1)
{
int bytesread = 0;
// Read the 32 bytes containing the GUID and length and stream_id info
get_sized_buffer(ctx->demux_ctx, cb, 32);
if(cb->buffer == NULL)
return CCX_EOF;
uint8_t guid[16];
memcpy(&guid, cb->buffer, 16); // Read the GUID
int x;
for(x=0; x<16; x++)
dbg_print(CCX_DMT_PARSE, "%02X ", guid[x]);
dbg_print(CCX_DMT_PARSE, "\n");
uint32_t len;
memcpy(&len, cb->buffer+16, 4); // Read the length
len-=32;
dbg_print(CCX_DMT_PARSE, "len %X\n", len);
uint32_t pad;
pad = len%8==0 ? 0 : 8- (len % 8); // Calculate the padding to add to the length
// to get to the next GUID
dbg_print(CCX_DMT_PARSE, "pad %X\n", pad);
uint32_t stream_id;
memcpy(&stream_id, cb->buffer+20, 4);
stream_id = stream_id & 0x7f; // Read and calculate the stream_id
dbg_print(CCX_DMT_PARSE, "stream_id: 0x%X\n", stream_id);
for(x=0; x<num_streams; x++)
dbg_print(CCX_DMT_PARSE, "video stream_id: %X\n", video_streams[x]);
if( !memcmp(guid, WTV_EOF, 16 ))
{
// This is the end of the data in this file
// read the padding at the end of the file
// and return one more byte
dbg_print(CCX_DMT_PARSE, "WTV EOF\n");
uint8_t *parsebuf;
parsebuf = (uint8_t*)malloc(1024);
do {
buffered_read(ctx->demux_ctx, parsebuf, 1024);
ctx->demux_ctx->past+=1024;
} while (result==1024);
ctx->demux_ctx->past+=result;
free(parsebuf);
free(cb->buffer);
cb->buffer=NULL;
//return one more byte so the final percentage is shown correctly
*(data->buffer+data->len)=0x00;
data->len++;
return CCX_EOF;
}
if( !memcmp(guid, WTV_STREAM2, 16 ) )
{
// The WTV_STREAM2 GUID appares near the start of the data dir
// It maps stream_ids to the type of stream
dbg_print(CCX_DMT_PARSE, "WTV STREAM2\n");
get_sized_buffer(ctx->demux_ctx, cb, 0xc+16);
if(cb->buffer==NULL)
return CCX_EOF;
static unsigned char stream_type[16];
memcpy(&stream_type, cb->buffer+0xc, 16); //Read the stream type GUID
const void *stream_guid;
if(ccx_options.wtvmpeg2)
stream_guid = WTV_STREAM_VIDEO; // We want mpeg2 data if the user set -wtvmpeg2
else
stream_guid = WTV_STREAM_MSTVCAPTION; // Otherwise we want the MSTV captions stream
if(!memcmp(stream_type, stream_guid, 16 ) )
{
video_streams[num_streams]=stream_id; // We keep a list of stream ids
num_streams++; // Even though there should only be 1
}
if (memcmp(stream_type, WTV_STREAM_AUDIO, 16))
alt_stream = stream_id;
len-=28;
}
if (!memcmp(guid, WTV_TIMING, 16) && ((use_alt_stream < WTV_CC_TIMESTAMP_MAGIC_THRESH && check_stream_id(stream_id, video_streams, num_streams))
|| (use_alt_stream == WTV_CC_TIMESTAMP_MAGIC_THRESH && stream_id == alt_stream)))
{
// The WTV_TIMING GUID contains a timestamp for the given stream_id
dbg_print(CCX_DMT_PARSE, "WTV TIMING\n");
get_sized_buffer(ctx->demux_ctx, cb, 0x8+0x8);
if(cb->buffer==NULL)
return CCX_EOF;
int64_t time;
memcpy(&time, cb->buffer+0x8, 8); // Read the timestamp
dbg_print(CCX_DMT_PARSE, "TIME: %ld\n", time);
if (time != -1 && time != WTV_CC_TIMESTAMP_MAGIC) { // Ignore -1 timestamps
set_current_pts(dec_ctx->timing, time_to_pes_time(time));
frames_since_ref_time = 0;
set_fts(dec_ctx->timing);
}
else if (time == WTV_CC_TIMESTAMP_MAGIC && stream_id != alt_stream) {
use_alt_stream++;
mprint("WARNING: %i WTV_CC_TIMESTAMP_MAGIC detected in cc timestamps. \n", use_alt_stream);
if (use_alt_stream == WTV_CC_TIMESTAMP_MAGIC_THRESH)
mprint("WARNING: WTV_CC_TIMESTAMP_MAGIC_THRESH reached. Switching to alt timestamps!\n");
}
len-=16;
}
if( !memcmp(guid, WTV_DATA, 16 )
&& check_stream_id(stream_id, video_streams, num_streams) && dec_ctx->timing->current_pts != 0
&& (ccx_options.wtvmpeg2 || (!ccx_options.wtvmpeg2 && len==2)))
{
// This is the data for a stream we want to process
dbg_print(CCX_DMT_PARSE, "\nWTV DATA\n");
get_sized_buffer(ctx->demux_ctx, cb, len);
if(cb->buffer==NULL)
return CCX_EOF;
memcpy(data->buffer+data->len, cb->buffer, len);
data->len+=result;
bytesread+=(int) len;
frames_since_ref_time++;
set_fts(dec_ctx->timing);
if(pad>0)
{ //Make sure we skip any padding too, since we are returning here
skip_sized_buffer(ctx->demux_ctx, cb, pad);
}
return bytesread;
}
if(len+pad>0)
{
// skip any remaining data
// For any unhandled GUIDs this will be len+pad
// For others it will just be pad
skip_sized_buffer(ctx->demux_ctx, cb, len+pad);
}
}
}
// read_header will read all the required information from
// the wtv header/root sections and calculate the skip_chunks.
// If successful, will return with the file positioned
// at the start of the data dir
int read_header(struct ccx_demuxer *ctx, struct wtv_chunked_buffer *cb)
{
ctx->startbytes_avail = (int)buffered_read_opt(ctx, ctx->startbytes, STARTBYTESLENGTH);
return_to_buffer(ctx, ctx->startbytes, ctx->startbytes_avail);
uint8_t *parsebuf;
parsebuf = (uint8_t*)malloc(1024);
buffered_read(ctx, parsebuf,0x42);
ctx->past+=result;
if (result!=0x42)
{
mprint("\nPremature end of file!\n");
return CCX_EOF;
}
// Expecting WTV header
if( !memcmp(parsebuf, WTV_HEADER, 16 ) )
{
dbg_print(CCX_DMT_PARSE, "\nWTV header\n");
}
else
{
mprint("\nMissing WTV header. Abort.\n");
return CCX_EOF;
}
//Next read just enough to get the location of the root directory
uint32_t filelen;
uint32_t root_dir;
memcpy(&filelen, parsebuf+0x30, 4);
dbg_print(CCX_DMT_PARSE, "filelen: %x\n", filelen);
memcpy(&root_dir, parsebuf+0x38, 4);
dbg_print(CCX_DMT_PARSE, "root_dir: %x\n", root_dir);
//Seek to start of the root dir. Typically 0x1100
buffered_skip(ctx,(root_dir*WTV_CHUNK_SIZE)-0x42);
ctx->past+=(root_dir*WTV_CHUNK_SIZE)-0x42;
if (result!=(root_dir*WTV_CHUNK_SIZE)-0x42)
return CCX_EOF;
// Read and calculate the meta data chunks in the file we need to skip over
// while parsing the file.
int end=0;
while(!end)
{
buffered_read(ctx, parsebuf, 32);
int x;
for(x=0; x<16; x++)
dbg_print(CCX_DMT_PARSE, "%02X ", parsebuf[x]);
dbg_print(CCX_DMT_PARSE, "\n");
if (result!=32)
{
mprint("\nPremature end of file!\n");
free(parsebuf);
return CCX_EOF;
}
ctx->past+=32;
if( !memcmp(parsebuf, WTV_EOF, 16 ))
{
dbg_print(CCX_DMT_PARSE, "WTV EOF\n");
end=1;
}
else
{
uint16_t len;
uint64_t file_length;
memcpy(&len, parsebuf+16, 2);
dbg_print(CCX_DMT_PARSE, "len: %x\n", len);
memcpy(&file_length, parsebuf+24, 8);
dbg_print(CCX_DMT_PARSE, "file_length: %x\n", file_length);
if(len>1024)
{
mprint("Too large for buffer!\n");
free(parsebuf);
return CCX_EOF;
}
buffered_read(ctx, parsebuf, len-32);
if (result!=len-32)
{
mprint("Premature end of file!\n");
free(parsebuf);
return CCX_EOF;
}
ctx->past+=len-32;
// Read a unicode string
uint32_t text_len;
memcpy(&text_len, parsebuf, 4); //text_len is number of unicode chars, not bytes.
dbg_print(CCX_DMT_PARSE, "text_len: %x\n", text_len);
char *string;
string = (char*)malloc(text_len+1); // alloc for ascii
string[text_len]='\0';
for(x=0; x<text_len; x++)
{
memcpy(&string[x], parsebuf+8+(x*2), 1); // unicode converted to ascii
}
dbg_print(CCX_DMT_PARSE, "string: %s\n", string);
// Ignore everything that doesn't contain the text ".entries."
if(strstr(string, WTV_TABLE_ENTRIES)!=NULL)
{
uint32_t value;
uint32_t flag;
memcpy(&value, parsebuf+(text_len*2)+8, 4);
memcpy(&flag, parsebuf+(text_len*2)+4+8, 4);
dbg_print(CCX_DMT_PARSE, "value: %x\n", value);
dbg_print(CCX_DMT_PARSE, "flag: %x\n", flag);
if(!add_skip_chunks(ctx, cb, value, flag))
{
mprint("Premature end of file!\n");
free(parsebuf);
return CCX_EOF;
}
}
free(string);
}
}
// Our list of skip_chunks is now complete
// Sort it (not sure if this is needed, but it doesn't hurt).
qsort(cb->skip_chunks, cb->count, sizeof(uint64_t), qsort_cmpint);
dbg_print(CCX_DMT_PARSE, "skip_chunks: ");
int x;
for(x=0; x<cb->count; x++)
dbg_print(CCX_DMT_PARSE, "%llx, ", cb->skip_chunks[x]);
dbg_print(CCX_DMT_PARSE, "\n");
// Seek forward to the start of the data dir
// Typically 0x40000
buffered_skip(ctx,(int)((cb->skip_chunks[cb->chunk]+WTV_META_CHUNK_SIZE)-ctx->past));
cb->filepos=(cb->skip_chunks[cb->chunk]+WTV_META_CHUNK_SIZE);
cb->chunk++;
ctx->past=cb->filepos;
free(parsebuf);
return CCX_OK;
}
// Return 1 for sucessfully read ts packet
int ts_readpacket(struct ccx_demuxer* ctx, struct ts_payload *payload)
{
unsigned int adaptation_field_length = 0;
unsigned int adaptation_field_control;
long long result;
if (ctx->m2ts)
{
/* M2TS just adds 4 bytes to each packet (so size goes from 188 to 192)
The 4 bytes are not important to us, so just skip
// TP_extra_header {
Copy_permission_indicator 2 unimsbf
Arrival_time_stamp 30 unimsbf
} */
unsigned char tp_extra_header[4];
result = buffered_read(ctx, tp_extra_header, 3);
ctx->past += result;
if (result != 4)
{
if (result>0)
mprint("Premature end of file!\n");
return CCX_EOF;
}
}
result = buffered_read(ctx, tspacket, 188);
ctx->past += result;
if (result != 188)
{
if (result > 0)
mprint("Premature end of file!\n");
return CCX_EOF;
}
int printtsprob = 1;
while (tspacket[0]!=0x47)
{
if (printtsprob)
{
dbg_print(CCX_DMT_DUMPDEF,"\nProblem: No TS header mark (filepos=%lld). Received bytes:\n", ctx->past);
dump(CCX_DMT_DUMPDEF, tspacket, 4, 0, 0);
dbg_print(CCX_DMT_DUMPDEF, "Skip forward to the next TS header mark.\n");
printtsprob = 0;
}
unsigned char *tstemp;
// The amount of bytes read into tspacket
int tslen = 188;
// Check for 0x47 in the remaining bytes of tspacket
tstemp = (unsigned char *) memchr (tspacket+1, 0x47, tslen-1);
if (tstemp != NULL )
{
// Found it
int atpos = tstemp-tspacket;
memmove (tspacket,tstemp,(size_t)(tslen-atpos));
result = buffered_read(ctx, tspacket+(tslen-atpos), atpos);
ctx->past+=result;
if (result!=atpos)
{
mprint("Premature end of file!\n");
return CCX_EOF;
}
}
else
{
// Read the next 188 bytes.
result = buffered_read(ctx, tspacket, tslen);
ctx->past+=result;
if (result!=tslen)
{
mprint("Premature end of file!\n");
return CCX_EOF;
}
}
}
payload->transport_error = (tspacket[1]&0x80)>>7;
payload->pesstart = (tspacket[1] & 0x40) >> 6;
// unsigned transport_priority = (tspacket[1]&0x20)>>5;
payload->pid = (((tspacket[1] & 0x1F) << 8) | tspacket[2]) & 0x1FFF;
// unsigned transport_scrambling_control = (tspacket[3]&0xC0)>>6;
adaptation_field_control = (tspacket[3]&0x30)>>4;
payload->counter = tspacket[3] & 0xF;
if (payload->transport_error)
{
dbg_print(CCX_DMT_DUMPDEF, "Warning: Defective (error indicator on) TS packet (filepos=%lld):\n", ctx->past);
dump(CCX_DMT_DUMPDEF, tspacket, 188, 0, 0);
}
payload->start = tspacket + 4;
payload->length = 188 - 4;
if ( adaptation_field_control & 2 )
{
// Take the PCR (Program Clock Reference) from here, in case PTS is not available (copied from telxcc).
adaptation_field_length = tspacket[4];
uint8_t af_pcr_exists = (tspacket[5] & 0x10) >> 4;
if (af_pcr_exists > 0 )
{
payload->pcr = 0;
payload->pcr |= (tspacket[6] << 25);
payload->pcr |= (tspacket[7] << 17);
payload->pcr |= (tspacket[8] << 9);
payload->pcr |= (tspacket[9] << 1);
payload->pcr |= (tspacket[10] >> 7);
/* Ignore 27 Mhz clock since we dont deal in nano seconds*/
//payload->pcr = ((tspacket[10] & 0x01) << 8);
//payload->pcr |= tspacket[11];
}
/* Read and evaluate the current video PES header. The function returns
* the length of the payload if successful, otherwise -1 is returned
* indicating a premature end of file / too small buffer.
* If sbuflen is
* 0 .. Read from file into nextheader
* >0 .. Use data in nextheader with the length of sbuflen
*/
int read_video_pes_header (unsigned char *nextheader, int *headerlength, int sbuflen)
{
// Read the next video PES
// ((nextheader[3]&0xf0)==0xe0)
unsigned peslen=nextheader[4]<<8 | nextheader[5];
unsigned payloadlength = 0; // Length of packet data bytes
if ( !sbuflen )
{
// Extension present, get it
buffered_read (nextheader+6,3);
past=past+result;
if (result!=3) {
// Consider this the end of the show.
return -1;
}
}
else
{
// We need at least 9 bytes to continue
if( sbuflen < 9 )
return -1;
}
*headerlength = 6+3;
unsigned hskip=0;
// Assume header[8] is right, but double check
if ( !sbuflen )
{
if (nextheader[8] > 0) {
buffered_read (nextheader+9,nextheader[8]);
past=past+result;
if (result!=nextheader[8]) {
return -1;
}
}
}
else
{
// See if the buffer is big enough
if( sbuflen < *headerlength + (int)nextheader[8] )
return -1;
}
*headerlength += (int) nextheader[8];
int falsepes = 0;
int pesext = 0;
// Avoid false positives, check --- not really needed
if ( (nextheader[7]&0xC0) == 0x80 ) {
// PTS only
hskip += 5;
if( (nextheader[9]&0xF1) != 0x21 || (nextheader[11]&0x01) != 0x01
|| (nextheader[13]&0x01) != 0x01 ) {
falsepes = 1;
printf("False PTS\n");
}
} else if ( (nextheader[7]&0xC0) == 0xC0 ) {
// PTS and DTS
hskip += 10;
if( (nextheader[9]&0xF1) != 0x31 || (nextheader[11]&0x01) != 0x01
|| (nextheader[13]&0x01) != 0x01
|| (nextheader[14]&0xF1) != 0x11 || (nextheader[16]&0x01) != 0x01
|| (nextheader[18]&0x01) != 0x01 ) {
falsepes = 1;
printf("False PTS/DTS\n");
}
} else if ( (nextheader[7]&0xC0) == 0x40 ) {
// Forbidden
falsepes = 1;
printf("False PTS/DTS flag\n");
}
if ( !falsepes && nextheader[7]&0x20 ) { // ESCR
if ((nextheader[9+hskip]&0xC4) != 0x04 || !(nextheader[11+hskip]&0x04)
|| !(nextheader[13+hskip]&0x04) || !(nextheader[14+hskip]&0x01) ) {
falsepes = 1;
printf("False ESCR\n");
}
hskip += 6;
}
if ( !falsepes && nextheader[7]&0x10 ) { // ES
if ( !(nextheader[9+hskip]&0x80) || !(nextheader[11+hskip]&0x01) ) {
printf("False ES\n");
falsepes = 1;
}
hskip += 3;
}
if ( !falsepes && nextheader[7]&0x04) { // add copy info
if ( !(nextheader[9+hskip]&0x80) ) {
printf("False add copy info\n");
falsepes = 1;
}
hskip += 1;
}
if ( !falsepes && nextheader[7]&0x02) { // PES CRC
hskip += 2;
}
if ( !falsepes && nextheader[7]&0x01) { // PES extension
if ( (nextheader[9+hskip]&0x0E)!=0x0E ) {
printf("False PES ext\n");
falsepes = 1;
}
hskip += 1;
pesext = 1;
}
if ( !falsepes ) {
hskip = nextheader[8];
}
if ( !falsepes && nextheader[7]&0x80 ) {
// Read PTS from byte 9,10,11,12,13
unsigned bits_9 = (nextheader[9] & 0x0E) << 29;
unsigned bits_10 = nextheader[10] << 22;
unsigned bits_11 = (nextheader[11] & 0xFE) << 14;
unsigned bits_12 = nextheader[12] << 7;
unsigned bits_13 = nextheader[13] >> 1;
current_pts = bits_9 | bits_10 | bits_11 | bits_12 | bits_13;
if (pts_set==0)
pts_set=1;
if (debug_verbose)
printf("Set PTS: %s (%u)\n",
print_mstime((current_pts)/(MPEG_CLOCK_FREQ/1000)),
unsigned(current_pts) );
/* The user data holding the captions seems to come between GOP and
* the first frame. The sync PTS (sync_pts) (set at picture 0)
* corresponds to the first frame of the current GOP. */
}
int main(int argc, char *argv[]) {
int argchar;
char* progname = argv[0];
sl* infns = sl_new(16);
char* outfnpat = NULL;
char* racol = "RA";
char* deccol = "DEC";
char* tempdir = "/tmp";
anbool gzip = FALSE;
sl* cols = sl_new(16);
int loglvl = LOG_MSG;
int nside = 1;
double margin = 0.0;
int NHP;
double md;
char* backref = NULL;
fitstable_t* intable;
fitstable_t** outtables;
char** myargs;
int nmyargs;
int i;
while ((argchar = getopt (argc, argv, OPTIONS)) != -1)
switch (argchar) {
case 'b':
backref = optarg;
break;
case 't':
tempdir = optarg;
break;
case 'c':
sl_append(cols, optarg);
break;
case 'g':
gzip = TRUE;
break;
case 'o':
outfnpat = optarg;
break;
case 'r':
racol = optarg;
break;
case 'd':
deccol = optarg;
break;
case 'n':
nside = atoi(optarg);
break;
case 'm':
margin = atof(optarg);
break;
case 'v':
loglvl++;
break;
case '?':
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
case 'h':
printHelp(progname);
return 0;
default:
return -1;
}
if (sl_size(cols) == 0) {
sl_free2(cols);
cols = NULL;
}
nmyargs = argc - optind;
myargs = argv + optind;
for (i=0; i<nmyargs; i++)
sl_append(infns, myargs[i]);
if (!sl_size(infns)) {
printHelp(progname);
printf("Need input filenames!\n");
exit(-1);
}
log_init(loglvl);
fits_use_error_system();
NHP = 12 * nside * nside;
logmsg("%i output healpixes\n", NHP);
outtables = calloc(NHP, sizeof(fitstable_t*));
assert(outtables);
md = deg2dist(margin);
/**
About the mincaps/maxcaps:
These have a center and radius-squared, describing the region
inside a small circle on the sphere.
The "mincaps" describe the regions that are definitely owned by a
single healpix -- ie, more than MARGIN distance from any edge.
That is, the mincap is the small circle centered at (0.5, 0.5) in
the healpix and with radius = the distance to the closest healpix
boundary, MINUS the margin distance.
Below, we first check whether a new star is within the "mincap"
of any healpix. If so, we stick it in that healpix and continue.
Otherwise, we check all the "maxcaps" -- these are the healpixes
it could *possibly* be in. We then refine with
healpix_within_range_of_xyz. The maxcap distance is the distance
to the furthest boundary point, PLUS the margin distance.
*/
cap_t* mincaps = malloc(NHP * sizeof(cap_t));
cap_t* maxcaps = malloc(NHP * sizeof(cap_t));
for (i=0; i<NHP; i++) {
// center
double r2;
double xyz[3];
double* cxyz;
double step = 1e-3;
double v;
double r2b, r2a;
cxyz = mincaps[i].xyz;
healpix_to_xyzarr(i, nside, 0.5, 0.5, mincaps[i].xyz);
memcpy(maxcaps[i].xyz, cxyz, 3 * sizeof(double));
logverb("Center of HP %i: (%.3f, %.3f, %.3f)\n", i, cxyz[0], cxyz[1], cxyz[2]);
// radius-squared:
// max is the easy one: max of the four corners (I assume)
r2 = 0.0;
healpix_to_xyzarr(i, nside, 0.0, 0.0, xyz);
logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
r2 = MAX(r2, distsq(xyz, cxyz, 3));
healpix_to_xyzarr(i, nside, 1.0, 0.0, xyz);
logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
r2 = MAX(r2, distsq(xyz, cxyz, 3));
healpix_to_xyzarr(i, nside, 0.0, 1.0, xyz);
logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
r2 = MAX(r2, distsq(xyz, cxyz, 3));
healpix_to_xyzarr(i, nside, 1.0, 1.0, xyz);
logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3));
r2 = MAX(r2, distsq(xyz, cxyz, 3));
logverb(" max distsq: %.3f\n", r2);
logverb(" margin dist: %.3f\n", md);
maxcaps[i].r2 = square(sqrt(r2) + md);
logverb(" max cap distsq: %.3f\n", maxcaps[i].r2);
r2a = r2;
r2 = 1.0;
r2b = 0.0;
for (v=0; v<=1.0; v+=step) {
healpix_to_xyzarr(i, nside, 0.0, v, xyz);
r2 = MIN(r2, distsq(xyz, cxyz, 3));
r2b = MAX(r2b, distsq(xyz, cxyz, 3));
healpix_to_xyzarr(i, nside, 1.0, v, xyz);
r2 = MIN(r2, distsq(xyz, cxyz, 3));
r2b = MAX(r2b, distsq(xyz, cxyz, 3));
healpix_to_xyzarr(i, nside, v, 0.0, xyz);
r2 = MIN(r2, distsq(xyz, cxyz, 3));
r2b = MAX(r2b, distsq(xyz, cxyz, 3));
healpix_to_xyzarr(i, nside, v, 1.0, xyz);
r2 = MIN(r2, distsq(xyz, cxyz, 3));
r2b = MAX(r2b, distsq(xyz, cxyz, 3));
}
mincaps[i].r2 = square(MAX(0, sqrt(r2) - md));
logverb("\nhealpix %i: min rad %g\n", i, sqrt(r2));
logverb("healpix %i: max rad %g\n", i, sqrt(r2a));
logverb("healpix %i: max rad(b) %g\n", i, sqrt(r2b));
assert(r2a >= r2b);
}
if (backref) {
fitstable_t* tab = fitstable_open_for_writing(backref);
int maxlen = 0;
char* buf;
for (i=0; i<sl_size(infns); i++) {
char* infn = sl_get(infns, i);
maxlen = MAX(maxlen, strlen(infn));
}
fitstable_add_write_column_array(tab, fitscolumn_char_type(), maxlen,
"filename", NULL);
fitstable_add_write_column(tab, fitscolumn_i16_type(), "index", NULL);
if (fitstable_write_primary_header(tab) ||
fitstable_write_header(tab)) {
ERROR("Failed to write header of backref table \"%s\"", backref);
exit(-1);
}
buf = malloc(maxlen+1);
assert(buf);
for (i=0; i<sl_size(infns); i++) {
char* infn = sl_get(infns, i);
int16_t ind;
memset(buf, 0, maxlen);
strcpy(buf, infn);
ind = i;
if (fitstable_write_row(tab, buf, &ind)) {
ERROR("Failed to write row %i of backref table: %s = %i",
i, buf, ind);
exit(-1);
}
}
if (fitstable_fix_header(tab) ||
fitstable_close(tab)) {
ERROR("Failed to fix header & close backref table");
exit(-1);
}
logmsg("Wrote backref table %s\n", backref);
free(buf);
}
for (i=0; i<sl_size(infns); i++) {
char* infn = sl_get(infns, i);
char* originfn = infn;
int r, NR;
tfits_type any, dubl;
il* hps = NULL;
bread_t* rowbuf;
int R;
char* tempfn = NULL;
char* padrowdata = NULL;
int ii;
logmsg("Reading input \"%s\"...\n", infn);
if (gzip) {
char* cmd;
int rtn;
tempfn = create_temp_file("hpsplit", tempdir);
asprintf_safe(&cmd, "gunzip -cd %s > %s", infn, tempfn);
logmsg("Running: \"%s\"\n", cmd);
rtn = run_command_get_outputs(cmd, NULL, NULL);
if (rtn) {
ERROR("Failed to run command: \"%s\"", cmd);
exit(-1);
}
free(cmd);
infn = tempfn;
}
intable = fitstable_open(infn);
if (!intable) {
ERROR("Couldn't read catalog %s", infn);
exit(-1);
}
NR = fitstable_nrows(intable);
logmsg("Got %i rows\n", NR);
any = fitscolumn_any_type();
dubl = fitscolumn_double_type();
fitstable_add_read_column_struct(intable, dubl, 1, 0, any, racol, TRUE);
fitstable_add_read_column_struct(intable, dubl, 1, sizeof(double), any, deccol, TRUE);
fitstable_use_buffered_reading(intable, 2*sizeof(double), 1000);
R = fitstable_row_size(intable);
rowbuf = buffered_read_new(R, 1000, NR, refill_rowbuffer, intable);
if (fitstable_read_extension(intable, 1)) {
ERROR("Failed to find RA and DEC columns (called \"%s\" and \"%s\" in the FITS file)", racol, deccol);
exit(-1);
}
for (r=0; r<NR; r++) {
int hp = -1;
double ra, dec;
int j;
double* rd;
void* rowdata;
void* rdata;
if (r && ((r % 100000) == 0)) {
logmsg("Reading row %i of %i\n", r, NR);
}
//printf("reading RA,Dec for row %i\n", r);
rd = fitstable_next_struct(intable);
ra = rd[0];
dec = rd[1];
logverb("row %i: ra,dec %g,%g\n", r, ra, dec);
if (margin == 0) {
hp = radecdegtohealpix(ra, dec, nside);
logverb(" --> healpix %i\n", hp);
} else {
double xyz[3];
anbool gotit = FALSE;
double d2;
if (!hps)
hps = il_new(4);
radecdeg2xyzarr(ra, dec, xyz);
for (j=0; j<NHP; j++) {
d2 = distsq(xyz, mincaps[j].xyz, 3);
if (d2 <= mincaps[j].r2) {
logverb(" -> in mincap %i (dist %g vs %g)\n", j, sqrt(d2), sqrt(mincaps[j].r2));
il_append(hps, j);
gotit = TRUE;
break;
}
}
if (!gotit) {
for (j=0; j<NHP; j++) {
d2 = distsq(xyz, maxcaps[j].xyz, 3);
if (d2 <= maxcaps[j].r2) {
logverb(" -> in maxcap %i (dist %g vs %g)\n", j, sqrt(d2), sqrt(maxcaps[j].r2));
if (healpix_within_range_of_xyz(j, nside, xyz, margin)) {
logverb(" -> and within range.\n");
il_append(hps, j);
}
}
}
}
//hps = healpix_rangesearch_radec(ra, dec, margin, nside, hps);
logverb(" --> healpixes: [");
for (j=0; j<il_size(hps); j++)
logverb(" %i", il_get(hps, j));
logverb(" ]\n");
}
//printf("Reading rowdata for row %i\n", r);
rowdata = buffered_read(rowbuf);
assert(rowdata);
j=0;
while (1) {
if (hps) {
if (j >= il_size(hps))
break;
hp = il_get(hps, j);
j++;
}
assert(hp < NHP);
assert(hp >= 0);
if (!outtables[hp]) {
char* outfn;
fitstable_t* out;
// MEMLEAK the output filename. You'll live.
asprintf_safe(&outfn, outfnpat, hp);
logmsg("Opening output file \"%s\"...\n", outfn);
out = fitstable_open_for_writing(outfn);
if (!out) {
ERROR("Failed to open output table \"%s\"", outfn);
exit(-1);
}
// Set the output table structure.
if (cols) {
fitstable_add_fits_columns_as_struct3(intable, out, cols, 0);
} else
fitstable_add_fits_columns_as_struct2(intable, out);
if (backref) {
tfits_type i16type;
tfits_type i32type;
// R = fitstable_row_size(intable);
int off = R;
i16type = fitscolumn_i16_type();
i32type = fitscolumn_i32_type();
fitstable_add_read_column_struct(out, i16type, 1, off,
i16type, "backref_file", TRUE);
off += sizeof(int16_t);
fitstable_add_read_column_struct(out, i32type, 1, off,
i32type, "backref_index", TRUE);
}
//printf("Output table:\n");
//fitstable_print_columns(out);
if (fitstable_write_primary_header(out) ||
fitstable_write_header(out)) {
ERROR("Failed to write output file headers for \"%s\"", outfn);
exit(-1);
}
outtables[hp] = out;
}
if (backref) {
int16_t brfile;
int32_t brind;
if (!padrowdata) {
padrowdata = malloc(R + sizeof(int16_t) + sizeof(int32_t));
assert(padrowdata);
}
// convert to FITS endian
brfile = htons(i);
brind = htonl(r);
// add backref data to rowdata
memcpy(padrowdata, rowdata, R);
memcpy(padrowdata + R, &brfile, sizeof(int16_t));
memcpy(padrowdata + R + sizeof(int16_t), &brind, sizeof(int32_t));
rdata = padrowdata;
} else {
rdata = rowdata;
}
if (cols) {
if (fitstable_write_struct_noflip(outtables[hp], rdata)) {
ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp);
}
} else {
if (fitstable_write_row_data(outtables[hp], rdata)) {
ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp);
}
}
if (!hps)
break;
}
if (hps)
il_remove_all(hps);
}
buffered_read_free(rowbuf);
// wack... buffered_read_free() just frees its internal buffer,
// not the "rowbuf" struct itself.
// who wrote this crazy code? Oh, me of 5 years ago. Jerk.
free(rowbuf);
fitstable_close(intable);
il_free(hps);
if (tempfn) {
logverb("Removing temp file %s\n", tempfn);
if (unlink(tempfn)) {
SYSERROR("Failed to unlink() temp file \"%s\"", tempfn);
}
tempfn = NULL;
}
// fix headers so that the files are valid at this point.
for (ii=0; ii<NHP; ii++) {
if (!outtables[ii])
continue;
off_t offset = ftello(outtables[ii]->fid);
if (fitstable_fix_header(outtables[ii])) {
ERROR("Failed to fix header for healpix %i after reading input file \"%s\"", ii, originfn);
exit(-1);
}
fseeko(outtables[ii]->fid, offset, SEEK_SET);
}
if (padrowdata) {
free(padrowdata);
padrowdata = NULL;
}
}
for (i=0; i<NHP; i++) {
if (!outtables[i])
continue;
if (fitstable_fix_header(outtables[i]) ||
fitstable_fix_primary_header(outtables[i]) ||
fitstable_close(outtables[i])) {
ERROR("Failed to close output table for healpix %i", i);
exit(-1);
}
}
free(outtables);
sl_free2(infns);
sl_free2(cols);
free(mincaps);
free(maxcaps);
return 0;
}
void* fitstable_next_struct(fitstable_t* tab) {
if (!tab->br) return NULL;
return buffered_read(tab->br);
}
// Return 1 for sucessfully read ts packet
int ts_readpacket(void)
{
buffered_read(tspacket,188);
past+=result;
if (result!=188)
{
if (result>0)
mprint("Premature end of file!\n");
end_of_file=1;
return 0;
}
int printtsprob = 1;
while (tspacket[0]!=0x47)
{
if (printtsprob)
{
mprint ("\nProblem: No TS header mark (filepos=%lld). Received bytes:\n", past);
dump (DMT_GENERIC_NOTICES, tspacket,4, 0, 0);
mprint ("Skip forward to the next TS header mark.\n");
printtsprob = 0;
}
unsigned char *tstemp;
// The amount of bytes read into tspacket
int tslen = 188;
// Check for 0x47 in the remaining bytes of tspacket
tstemp = (unsigned char *) memchr (tspacket+1, 0x47, tslen-1);
if (tstemp != NULL )
{
// Found it
int atpos = tstemp-tspacket;
memmove (tspacket,tstemp,(size_t)(tslen-atpos));
buffered_read(tspacket+(tslen-atpos),atpos);
past+=result;
if (result!=atpos)
{
mprint("Premature end of file!\n");
end_of_file=1;
return 0;
}
}
else
{
// Read the next 188 bytes.
buffered_read(tspacket,tslen);
past+=result;
if (result!=tslen)
{
mprint("Premature end of file!\n");
end_of_file=1;
return 0;
}
}
}
unsigned char *payload_start = tspacket + 4;
unsigned payload_length = 188 - 4;
unsigned transport_error_indicator = (tspacket[1]&0x80)>>7;
unsigned payload_start_indicator = (tspacket[1]&0x40)>>6;
// unsigned transport_priority = (tspacket[1]&0x20)>>5;
unsigned pid = (((tspacket[1] & 0x1F) << 8) | tspacket[2]) & 0x1FFF;
// unsigned transport_scrambling_control = (tspacket[3]&0xC0)>>6;
unsigned adaptation_field_control = (tspacket[3]&0x30)>>4;
unsigned ccounter = tspacket[3] & 0xF;
if (transport_error_indicator)
{
mprint ("Warning: Defective (error indicator on) TS packet (filepos=%lld):\n", past);
dump (DMT_GENERIC_NOTICES, tspacket, 188, 0, 0);
}
unsigned adaptation_field_length = 0;
if ( adaptation_field_control & 2 )
{
// Take the PCR (Program Clock Reference) from here, in case PTS is not available (copied from telxcc).
adaptation_field_length = tspacket[4];
uint8_t af_pcr_exists = (tspacket[5] & 0x10) >> 4;
if (af_pcr_exists > 0) {
uint64_t pts = 0;
pts |= (tspacket[6] << 25);
pts |= (tspacket[7] << 17);
pts |= (tspacket[8] << 9);
pts |= (tspacket[9] << 1);
pts |= (tspacket[10] >> 7);
global_timestamp = (uint32_t) pts / 90;
pts = ((tspacket[10] & 0x01) << 8);
pts |= tspacket[11];
global_timestamp += (uint32_t) (pts / 27000);
if (!global_timestamp_inited)
{
min_global_timestamp = global_timestamp;
global_timestamp_inited = 1;
}
}
int
main(int argc, char *argv[])
{
char *load_file = (char *)NULL; /* DB load path/file name */
char *optstring; /* Command option string */
char *logfile = NULL; /* Log file name for samfsrestore -g option */
struct sam_stat sb;
int option; /* Current getopt option being processed */
int io_sz, read_size;
int len;
operator_t operator; /* Operator data */
shm_alloc_t master_shm; /* Master shared memory structure */
program_name = basename(argv[0]);
operation = NONE;
replace_newer = false;
replace = false;
swapped = false;
verbose = false;
quiet = false;
noheaders = false;
strip_slashes = false;
list_by_inode = false;
online_data = false;
partial_data = false;
scan_only = false;
unarchived_data = false;
use_file_list = false;
read_buffer_size = write_buffer_size = CSD_DEFAULT_BUFSZ;
block_size = 0;
memset(&csd_header, '\0', sizeof (csd_header));
CustmsgInit(0, NULL);
if ((Initial_path = getcwd(NULL, (MAXPATHLEN + 1))) == NULL) {
error(1, errno, catgets(catfd, SET, 590, "cannot getcwd()"));
}
/*
* If basename is samfsrestore, fake out "r" option; if
* samfsdump, fake out "c" option. Allow redundant specification
* specifications like "samfsrestore r", "samfsdump c". Allow
* "samfsrestore t" to override "r" option so that we look
* like "ufsrestore t". If basename is none of the above, then
* user must specify one of c,r,t,O.
*/
if (strcmp(program_name, "samfsrestore") == 0) {
operation = RESTORE;
optstring = RESTORE_OPT;
qfs = false;
} else if (strcmp(program_name, "samfsdump") == 0) {
operation = DUMP;
optstring = DUMP_OPT;
qfs = false;
} else if (strcmp(program_name, "qfsdump") == 0) {
operation = DUMP;
optstring = QFSDUMP_OPT;
qfs = true;
} else if (strcmp(program_name, "qfsrestore") == 0) {
operation = RESTORE;
optstring = RESTORE_OPT;
qfs = true;
}
if (operation == NONE) {
error(1, 0, catgets(catfd, SET, 13508,
"Improper invocation of SAM-FS csd "
"utility %s"),
program_name);
}
for (nexcluded = CSD_MAX_EXCLUDED - 1; nexcluded >= 0; nexcluded--) {
excluded[nexcluded] = NULL;
}
nexcluded = 0;
for (nincluded = CSD_MAX_INCLUDED - 1; nincluded >= 0; nincluded--) {
included[nincluded] = NULL;
}
nincluded = 0;
while ((option = getopt(argc, argv, optstring)) != EOF) {
switch (option) {
case 'B': /* select buffer size */
io_sz = atoi(optarg) * 512;
if (io_sz < CSD_MIN_BUFSZ || io_sz > CSD_MAX_BUFSZ) {
error(1, 0,
catgets(catfd, SET, 13509,
"Invalid buffer size, must "
"be >= %d and <= %d, "
"specified in 512b units"),
CSD_MIN_BUFSZ/512, CSD_MAX_BUFSZ/512);
}
read_buffer_size = write_buffer_size = io_sz;
break;
case 'b': /* select blocking factor */
io_sz = atoi(optarg) * 512;
if (io_sz < 0 || io_sz > CSD_MAX_BUFSZ) {
error(1, 0, catgets(catfd, SET, 13510,
"Invalid blocking factor, "
"must be >= 0 and <= %d, "
"specified in 512b units"),
CSD_MAX_BUFSZ/512);
}
block_size = io_sz;
break;
case 'd': /* enable debug messages */
debugging = 1;
break;
case 'D': /* enable directio */
Directio++;
break;
case 'I': /* file of paths to process */
if (nincluded > CSD_MAX_INCLUDED) {
error(1, 0, catgets(catfd, SET, 13537,
"Maximum -I (include file) entries exceeded, max. is %d."),
CSD_MAX_INCLUDED);
}
included[nincluded] = optarg;
len = strlen(optarg);
if (len > 0) {
*(included[nincluded] + len) = '\0';
nincluded++;
} else {
error(1, 0, catgets(catfd, SET, 13538,
"-I (include file) entry "
"<%s> invalid"),
optarg);
}
break;
case 'X': /* excluded directory list */
if (nexcluded > CSD_MAX_EXCLUDED) {
error(1, 0, catgets(catfd, SET, 13520,
"Maximum -X (excluded "
"directory) entries "
"exceeded, max. is %d."),
CSD_MAX_EXCLUDED);
}
excluded[nexcluded] = optarg;
len = strlen(optarg);
if (len > 0) {
*(excluded[nexcluded] + len) = '\0';
nexcluded++;
} else {
error(1, 0, catgets(catfd, SET, 13521,
"-X (excluded directory) entry <%s> "
"invalid"),
optarg);
}
break;
case 'f': /* specify dump file */
dump_file = optarg;
break;
case 'g': /* specify log file */
logfile = optarg;
break;
case 'H': /* no headers */
noheaders = true;
break;
case 'i': /* print inode numbers */
ls_options |= LS_INODES;
break;
case 'l': /* print 1 line ls */
verbose = true;
ls_options |= LS_LINE1;
break;
case 'n': /* Deprecated. Do not break script. */
break;
case 'P': /* dump partial online data */
partial_data = true;
break;
case 'q': /* Do not report damage warnings */
quiet = true;
break;
case 'r': /* replace file if dump is newer */
replace_newer = true;
break;
case 'R': /* replace file */
replace = true;
break;
case 's': /* strip leading /'s */
strip_slashes = true;
break;
case 'S': /* scan only */
scan_only = true;
break;
case 't': /* list dump file */
if (operation & DUMP) {
error(1, 0,
catgets(catfd, SET, 941,
"Do not specify 't' option."));
}
/* samfsrestore t is LIST, not RESTORE */
operation = LIST;
break;
case 'T': /* statistics mode */
statistics = true;
break;
case 'u': /* dump unarchived data */
unarchived_data = true;
break;
case 'U': /* dump online data */
online_data = true;
break;
case 'v': /* verbose mode */
verbose = true;
break;
case 'W':
/* warning mode, allowed for historical reasons */
break;
case '2': /* print 2 line ls */
verbose = true;
ls_options |= LS_LINE2;
break;
case 'Y': /* file list for dump */
use_file_list = true;
break;
case 'Z': /* generate database load file */
load_file = optarg;
break;
case '?':
usage(); /* doesn't return */
break;
}
}
if (scan_only) { /* clear dump related flags */
online_data = partial_data = unarchived_data = false;
}
if (block_size && (block_size > read_buffer_size)) {
block_size = read_buffer_size;
}
if (operation == DUMP && use_file_list) {
operation = LISTDUMP;
}
if (debugging) {
int i;
fprintf(stderr, "\nCOMMAND ARGUMENT VALUES\n");
fprintf(stderr, "operation = 0x%x\n", operation);
fprintf(stderr, "debugging = %d\n", debugging);
fprintf(stderr, "dump_file = %s\n",
((dump_file != (char *)NULL) ? dump_file : "UNDEFINED"));
fprintf(stderr, "logfile = %s\n",
((logfile != (char *)NULL) ? logfile : "UNDEFINED"));
fprintf(stderr, "noheaders = %d\n", noheaders);
fprintf(stderr, "verbose = %d\n", verbose);
fprintf(stderr, "quiet = %d\n", quiet);
fprintf(stderr, "qfs = %d\n", qfs);
fprintf(stderr, "replace = %d\n", replace);
fprintf(stderr, "replace_newer = %d\n", replace_newer);
fprintf(stderr, "strip_slashes = %d\n", strip_slashes);
fprintf(stderr, "ls_options = 0x%x\n", ls_options);
fprintf(stderr, "load_file = %s\n",
((load_file != (char *)NULL) ? load_file : "UNDEFINED"));
fprintf(stderr, "online_data = %d\n", online_data);
fprintf(stderr, "partial_data = %d\n", partial_data);
fprintf(stderr, "scan_only = %d\n", scan_only);
fprintf(stderr, "unarchived_data = %d\n", unarchived_data);
fprintf(stderr, "use_file_list = %d\n", use_file_list);
fprintf(stderr, "read buffer size = %ld\n", read_buffer_size);
fprintf(stderr, "write buffer size = %ld\n",
write_buffer_size);
fprintf(stderr, "blocking factor = %ld\n", block_size);
for (i = optind; i < argc; i++) {
fprintf(stderr, "File argument %d = %s\n",
(i - optind + 1), argv[i]);
}
fprintf(stderr, "END COMMAND ARGUMENT VALUES\n\n");
}
/*
* If dump file has not been specified, exit with error and usage
*/
if ((!scan_only) && dump_file == (char *)NULL) {
error(0, 0,
catgets(catfd, SET, 5019,
"%s: Dump file name not specified (-f)"),
program_name);
usage();
}
/*
* Access master shared memory segment
*/
if ((master_shm.shared_memory = (shm_ptr_tbl_t *)
sam_mastershm_attach(0, SHM_RDONLY)) == (void *)-1) {
/*
* If shared memory not available, effective ID must be
* super-user.
*/
if (geteuid() != 0) {
error(1, 0,
catgets(catfd, SET, 100,
"%s may be run only by super-user.\n"),
program_name);
}
} else {
/*
* If operator is not super-user, issue error message and exit
*/
SET_SAM_OPER_LEVEL(operator);
shmdt((char *)master_shm.shared_memory);
if (!SAM_ROOT_LEVEL(operator)) {
error(1, 0,
catgets(catfd, SET, 100,
"%s may be run only by super-user.\n"),
program_name);
}
}
/*
* Open dump file and process header
*/
switch (operation) {
case NONE:
error(1, 0,
catgets(catfd, SET, 1845,
"One of c, r, O or t options must be specified."));
usage(); /* doesn't return */
break;
case DUMP:
case LISTDUMP:
if (!scan_only) {
if (strcmp(dump_file, "-") == 0) {
CSD_fd = dup(STDOUT);
} else {
CSD_fd = open(dump_file,
O_WRONLY|O_TRUNC|O_CREAT|SAM_O_LARGEFILE,
0666 & ~umask(0));
}
close(STDOUT);
if (CSD_fd < 0) {
error(1, errno,
catgets(catfd, SET, 216,
"%s: Cannot create dump file"),
dump_file);
}
}
break;
case RESTORE:
case LIST:
if (0 == strcmp(dump_file, "-")) {
CSD_fd = dup(STDIN);
} else {
CSD_fd = open(dump_file, O_RDONLY | SAM_O_LARGEFILE);
}
close(STDIN);
if (CSD_fd < 0) {
error(1, errno,
catgets(catfd, SET, 225,
"%s: Cannot open dump file"),
dump_file);
}
if (noheaders) {
break;
}
if (buffered_read(CSD_fd, &csd_header, sizeof (csd_hdr_t)) !=
sizeof (csd_hdr_t)) {
error(1, errno,
catgets(catfd, SET, 244,
"%s: Header record read error"),
dump_file);
}
if (csd_header.csd_header.magic != CSD_MAGIC) {
if (csd_header.csd_header.magic != CSD_MAGIC_RE) {
error(1, 0,
catgets(catfd, SET, 13529,
"%s: Volume is not in dump format."),
dump_file);
} else {
error(0, 0,
"Dump file was generated on an "
"opposite endian machine");
swapped = true;
sam_byte_swap(csd_header_swap_descriptor,
&csd_header, sizeof (csd_hdr_t));
}
}
csd_version = csd_header.csd_header.version;
switch (csd_version) {
case CSD_VERS_6:
case CSD_VERS_5:
/* read remainder of extended header */
io_sz = sizeof (csd_hdrx_t) - sizeof (csd_hdr_t);
read_size = buffered_read(CSD_fd,
(char *)((char *)(&csd_header) +
sizeof (csd_hdr_t)), io_sz);
if (swapped) {
sam_bswap4(&csd_header.csd_header_flags, 1);
sam_bswap4(&csd_header.csd_header_magic, 1);
sam_bswap8(&csd_header.csd_fs_magic, 1);
}
if ((read_size != io_sz) ||
csd_header.csd_header_magic != CSD_MAGIC) {
error(1, errno,
catgets(catfd, SET, 244,
"%s: Header record read error"),
dump_file);
}
break;
case CSD_VERS_4:
case CSD_VERS_3:
case CSD_VERS_2:
break;
default:
error(1, 0,
catgets(catfd, SET, 5013,
"%s: Header record error: Bad version number (%d)"),
dump_file, csd_version);
}
if (verbose) {
printf(catgets(catfd, SET, 972, "Dump created:%s"),
ctime((const time_t *)&csd_header.csd_header.time));
}
if (debugging) {
fprintf(stderr, "Dump created:%s\n",
ctime((const time_t *)&csd_header.csd_header.time));
}
}
if ((logfile != NULL) && (log_st = fopen(logfile, "w")) == NULL) {
error(0, errno, "%s", logfile);
error(0, 0, catgets(catfd, SET, 1856,
"Open failed on (%s)"), logfile);
}
if ((load_file != NULL)) {
if (0 == strcmp(load_file, "-")) {
DB_FILE = fdopen(dup(STDOUT), "w");
close(STDOUT);
} else if ((DB_FILE = fopen64(load_file, "w")) == NULL) {
error(0, errno, "%s", load_file);
error(1, 0, catgets(catfd, SET, 1856,
"Open failed on (%s)"), load_file);
}
}
switch (operation) {
case NONE:
break;
case DUMP: {
char *filename;
if (debugging) {
fprintf(stderr, "dumping. argc %d, initial "
"path '%s' \n",
optind, Initial_path);
}
while ((filename = get_path(argc, argv)) != NULL) {
if (debugging) {
fprintf(stderr, "dumping path '%s' \n",
filename);
}
/*
* Try to make sure we don't dump from a non-SAM-FS
* filesystem by calling sam_stat() and checking the
* attributes.
*/
if (sam_stat(filename, &sb, sizeof (sb)) < 0 ||
! SS_ISSAMFS(sb.attr)) {
BUMP_STAT(errors);
BUMP_STAT(errors_dir);
error(1, errno,
catgets(catfd, SET, 259,
"%s: Not a SAM-FS file."),
filename);
} else {
if (SAM_fd != -1) {
close(SAM_fd);
}
SAM_fd = open_samfs(filename);
csd_dump_path("initial", filename,
(mode_t)sb.st_mode);
if (S_ISDIR(sb.st_mode)) {
process_saved_dir_list(filename);
}
}
if (chdir(Initial_path) < 0) {
error(1, errno,
catgets(catfd, SET, 212,
"%s: cannot chdir()"),
Initial_path);
}
}
}
bflush(CSD_fd);
break;
case LISTDUMP: {
char *filename;
if (debugging) {
fprintf(stderr, "dumping. argc %d, initial "
"path '%s' \n",
optind, Initial_path);
}
while ((filename = get_path(argc, argv)) != NULL) {
if (debugging) {
fprintf(stderr, "dumping path '%s' \n",
filename);
}
if (strcmp(filename, "-") == 0) {
DL_FILE = fdopen(dup(STDIN), "r");
close(STDIN);
} else if ((DL_FILE = fopen(filename, "r")) == NULL) {
error(0, errno, "%s", load_file);
error(1, 0, catgets(catfd, SET, 1856,
"Open failed on (%s)"), load_file);
}
if (SAM_fd != -1) {
close(SAM_fd);
}
csd_dump_files(DL_FILE, filename);
fclose(DL_FILE);
DL_FILE = NULL;
}
}
bflush(CSD_fd);
break;
case RESTORE:
umask(0);
cs_restore(strip_slashes, (argc - optind), &argv[optind]);
break;
case LIST:
cs_list((argc - optind), &argv[optind]);
}
close(CSD_fd);
if (statistics) {
print_stats();
}
return (exit_status);
}
int replay_log_record(HttpHandlerContext * const context,
BufferedReadContext * const brc)
{
DBLog * const db_log = &app_context.db_log;
if (db_log->db_log_fd == -1) {
return 1;
}
char buf_cookie_head[sizeof DB_LOG_RECORD_COOKIE_HEAD - (size_t) 1U];
char buf_cookie_tail[sizeof DB_LOG_RECORD_COOKIE_TAIL - (size_t) 1U];
char buf_number[50];
char *pnt;
char *endptr;
const off_t current_offset = lseek(db_log->db_log_fd, (off_t) 0, SEEK_CUR);
ssize_t readnb = buffered_read(brc, buf_cookie_head,
sizeof buf_cookie_head);
if (readnb == (ssize_t) 0) {
return 1;
}
if (readnb != (ssize_t) sizeof buf_cookie_head ||
memcmp(buf_cookie_head, DB_LOG_RECORD_COOKIE_HEAD, readnb) != 0) {
if (lseek(db_log->db_log_fd, current_offset, SEEK_SET) == (off_t) -1 ||
ftruncate(db_log->db_log_fd, current_offset) != 0) {
}
return -1;
}
pnt = buf_number;
for (;;) {
if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U) {
return -1;
}
if (*pnt == ' ') {
break;
}
if (++pnt == &buf_number[sizeof buf_number]) {
return -1;
}
}
if (*buf_number == *DB_LOG_RECORD_COOKIE_MARK_CHAR) {
time_t ts = (time_t) strtol(buf_number + 1U, &endptr, 16);
if (endptr == NULL || endptr == buf_number + 1U || ts <= (time_t) 0) {
return -1;
}
char t;
if (buffered_read(brc, &t, (size_t) 1U) != (size_t) 1U ||
t != *DB_LOG_RECORD_COOKIE_TIMESTAMP_CHAR) {
return -1;
}
readnb = buffered_read(brc, buf_cookie_tail, sizeof buf_cookie_tail);
if (readnb != (ssize_t) sizeof buf_cookie_tail ||
memcmp(buf_cookie_tail, DB_LOG_RECORD_COOKIE_TAIL, readnb) != 0) {
return -1;
}
return 0;
}
int verb = (int) strtol(buf_number, &endptr, 16);
if (endptr == NULL || endptr == buf_number) {
return -1;
}
pnt = buf_number;
for (;;) {
if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U) {
return -1;
}
if (*pnt == ':') {
break;
}
if (++pnt == &buf_number[sizeof buf_number]) {
return -1;
}
}
size_t uri_len = (size_t) strtoul(buf_number, &endptr, 16);
if (endptr == NULL || endptr == buf_number || uri_len <= (size_t) 0U) {
return -1;
}
char *uri;
if ((uri = malloc(uri_len + (size_t) 1U)) == NULL) {
_exit(1);
}
if (buffered_read(brc, uri, uri_len) != (ssize_t) uri_len) {
return -1;
}
*(uri + uri_len) = 0;
pnt = buf_number;
if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U ||
*pnt != ' ') {
free(uri);
return -1;
}
for (;;) {
if (buffered_read(brc, pnt, (size_t) 1U) != (ssize_t) 1U) {
free(uri);
return -1;
}
if (*pnt == ':') {
break;
}
if (++pnt == &buf_number[sizeof buf_number]) {
free(uri);
return -1;
}
}
size_t body_len = (size_t) strtoul(buf_number, &endptr, 16);
if (endptr == NULL || endptr == buf_number) {
free(uri);
return -1;
}
char *body = NULL;
if (body_len > (size_t) 0U) {
body = malloc(body_len + (size_t) 1U);
if (body == NULL) {
free(uri);
_exit(1);
}
}
if (body != NULL) {
assert(body_len > (size_t) 0U);
if (buffered_read(brc, body, body_len) != (ssize_t) body_len) {
free(uri);
free(body);
return -1;
}
*(body + body_len) = 0;
}
readnb = buffered_read(brc, buf_cookie_tail, sizeof buf_cookie_tail);
if (readnb != (ssize_t) sizeof buf_cookie_tail ||
memcmp(buf_cookie_tail, DB_LOG_RECORD_COOKIE_TAIL, readnb) != 0) {
free(body);
free(uri);
return -1;
}
fake_request(context, verb, uri, body, body_len, 1, 0);
free(body);
free(uri);
return 0;
}
/**
* Read from a socket. (Use this instead of read() because it has
* buffering.)
* @param s Socket
* @param buf Buffer to get
* @param len Length
* @return int
*/
int sread(ano_socket_t s, char *buf, int len)
{
return buffered_read(s, buf, len);
}
// Return 1 for sucessfully read ts packet
int ts_readpacket(void)
{
buffered_read(tspacket,188);
past+=result;
if (result!=188)
{
if (result>0)
mprint("Premature end of file!\n");
end_of_file=1;
return 0;
}
int printtsprob = 1;
while (tspacket[0]!=0x47)
{
if (printtsprob)
{
mprint ("\nProblem: No TS header mark. Received bytes:\n");
dump (DMT_GENERIC_NOTICES, tspacket,4, 0, 0);
mprint ("Skip forward to the next TS header mark.\n");
printtsprob = 0;
}
unsigned char *tstemp;
// The amount of bytes read into tspacket
int tslen = 188;
// Check for 0x47 in the remaining bytes of tspacket
tstemp = (unsigned char *) memchr (tspacket+1, 0x47, tslen-1);
if (tstemp != NULL )
{
// Found it
int atpos = tstemp-tspacket;
memmove (tspacket,tstemp,(size_t)(tslen-atpos));
buffered_read(tspacket+(tslen-atpos),atpos);
past+=result;
if (result!=atpos)
{
mprint("Premature end of file!\n");
end_of_file=1;
return 0;
}
}
else
{
// Read the next 188 bytes.
buffered_read(tspacket,tslen);
past+=result;
if (result!=tslen)
{
mprint("Premature end of file!\n");
end_of_file=1;
return 0;
}
}
}
unsigned char *payload_start = tspacket + 4;
unsigned payload_length = 188 - 4;
unsigned transport_error_indicator = (tspacket[1]&0x80)>>7;
unsigned payload_start_indicator = (tspacket[1]&0x40)>>6;
// unsigned transport_priority = (tspacket[1]&0x20)>>5;
unsigned pid = (((tspacket[1] & 0x1F) << 8) | tspacket[2]) & 0x1FFF;
// unsigned transport_scrambling_control = (tspacket[3]&0xC0)>>6;
unsigned adaptation_field_control = (tspacket[3]&0x30)>>4;
unsigned ccounter = tspacket[3] & 0xF;
if (transport_error_indicator)
{
mprint ("Warning: Defective (error indicator on) TS packet:\n");
dump (DMT_GENERIC_NOTICES, tspacket, 188, 0, 0);
}
unsigned adaptation_field_length = 0;
if ( adaptation_field_control & 2 )
{
// TODO: If present, we should take the PCR (Program Clock Reference) from here, in case PTS is not
// available, as done in telxcc.
adaptation_field_length = tspacket[4];
payload_start = payload_start + adaptation_field_length + 1;
payload_length = tspacket+188-payload_start;
}
dbg_print(DMT_PARSE, "TS pid: %d PES start: %d counter: %u payload length: %u adapt length: %d\n",
pid, payload_start_indicator, ccounter, payload_length,
int(adaptation_field_length));
// Catch bad packages with adaptation_field_length > 184 and
// the unsigned nature of payload_length leading to huge numbers.
if (payload_length > 184)
{
// This renders the package invalid
payload_length = 0;
dbg_print(DMT_PARSE, " Reject package - set length to zero.\n");
}
// Save data in global struct
payload.start = payload_start;
payload.length = payload_length;
payload.pesstart = payload_start_indicator;
payload.pid = pid;
payload.counter = ccounter;
payload.transport_error = transport_error_indicator;
if (payload_length == 0)
{
dbg_print(DMT_PARSE, " No payload in package.\n");
}
// Store packet information
return 1;
}