int main(int argc, char **argv)
{
const int MaxFrameSize = 16*MAX_VDIF_FRAME_BYTES;
char buffer[MaxFrameSize];
FILE * input;
int readbytes, framebytes, framemjd, framesecond, framenumber, frameinvalid, datambps, framespersecond;
long long framesread;
vdif_header *header;
if(argc != 3)
{
usage();
return EXIT_FAILURE;
}
input = fopen(argv[1], "r");
if(input == NULL)
{
fprintf(stderr, "Cannot open input file %s\n", argv[1]);
exit(EXIT_FAILURE);
}
datambps = atoi(argv[2]);
readbytes = fread(buffer, 1, MaxFrameSize, input); //read the VDIF header
header = (vdif_header*)buffer;
framebytes = getVDIFFrameBytes(header);
if(framebytes > MaxFrameSize) {
fprintf(stderr, "Cannot read frame with %d bytes > max (%d), formal max %d\n", framebytes, MaxFrameSize, MAX_VDIF_FRAME_BYTES);
exit(EXIT_FAILURE);
}
framemjd = getVDIFFrameMJD(header);
framesecond = getVDIFFrameSecond(header);
framenumber = getVDIFFrameNumber(header);
frameinvalid = getVDIFFrameInvalid(header);
framespersecond = (int)((((long long)datambps)*1000000)/(8*(framebytes-VDIF_HEADER_BYTES)));
printf("Frames per second is %d\n", framespersecond);
rewind(input); //go back to the start
framesread = 0;
while(!feof(input)) {
readbytes = fread(buffer, 1, framebytes, input); //read the whole VDIF packet
if (readbytes < framebytes) {
fprintf(stderr, "Header read failed - probably at end of file.\n");
break;
}
header = (vdif_header*)buffer;
framemjd = getVDIFFrameMJD(header);
framesecond = getVDIFFrameSecond(header);
framenumber = getVDIFFrameNumber(header);
frameinvalid = getVDIFFrameInvalid(header);
printf("MJD is %d, second is %d, framenumber is %d, frameinvalid is %d\n", framemjd, framesecond, framenumber, frameinvalid);
printf("Threadid is %d, stationid is %d, numbits is %d\n", getVDIFThreadID(header), getVDIFStationID(header), getVDIFBitsPerSample(header));
if(getVDIFFrameBytes(header) != framebytes) {
fprintf(stderr, "Framebytes has changed! Can't deal with this, aborting\n");
break;
}
framesread++;
}
printf("Read %lld frames\n", framesread);
fclose(input);
return EXIT_SUCCESS;
}
int main(int argc, char **argv)
{
int VERBOSE = 0;
int FORCE_VALID = 1;
char buffer[MAX_VDIF_FRAME_BYTES];
FILE * input;
FILE * output;
int readbytes, framebytes, framemjd, framesecond, framenumber, frameinvalid, datambps, framespersecond;
int nextmjd, nextsecond, nextnumber;
int overwritemjd, overwritesecond, overwritenumber;
int offsetmjd=0, offsetsecond=0, offsetnumber=0;
long long framesread, frameswrote;
vdif_header *header;
if(argc != 4 && argc != 5)
{
usage();
return EXIT_FAILURE;
}
overwritemjd = 0;
overwritesecond = 0;
overwritenumber = 0;
if(argc == 5) {
overwritemjd = atoi(argv[4]);
overwritesecond = (int)(86400.0*(atof(argv[4]) - overwritemjd));
}
input = fopen(argv[1], "r");
if(input == NULL)
{
fprintf(stderr, "Cannot open input file %s\n", argv[1]);
exit(EXIT_FAILURE);
}
output = fopen(argv[2], "w");
if(output == NULL)
{
fprintf(stderr, "Cannot open output file %s\n", argv[2]);
exit(EXIT_FAILURE);
}
if(FORCE_VALID) {
printf("Forcing all read frames to be VALID! (Temp workaround for invalid tagged EVLA VDIF data)\n");
}
datambps = atoi(argv[3]);
readbytes = fread(buffer, 1, VDIF_HEADER_BYTES, input); //read the VDIF header
header = (vdif_header*)buffer;
framebytes = getVDIFFrameBytes(header);
if(framebytes > MAX_VDIF_FRAME_BYTES) {
fprintf(stderr, "Cannot read frame with %d bytes > max (%d)\n", framebytes, MAX_VDIF_FRAME_BYTES);
exit(EXIT_FAILURE);
}
nextmjd = getVDIFFrameMJD(header);
nextsecond = getVDIFFrameSecond(header);
nextnumber = getVDIFFrameNumber(header);
if(overwritemjd > 0) {
offsetmjd = overwritemjd - nextmjd;
offsetsecond = overwritesecond - nextsecond;
offsetnumber = overwritenumber - nextnumber;
}
framespersecond = (int)((((long long)datambps)*1000000)/(8*(framebytes-VDIF_HEADER_BYTES)));
printf("Frames per second is %d\n", framespersecond);
rewind(input); //go back to the start
framesread = 0;
frameswrote = 0;
while(!feof(input)) {
readbytes = fread(buffer, 1, framebytes, input); //read the whole VDIF packet
if (readbytes < framebytes) {
fprintf(stderr, "Header read failed - probably at end of file.\n");
break;
}
header = (vdif_header*)buffer;
framemjd = getVDIFFrameMJD(header);
framesecond = getVDIFFrameSecond(header);
framenumber = getVDIFFrameNumber(header);
frameinvalid = getVDIFFrameInvalid(header);
if(getVDIFFrameBytes(header) != framebytes) {
fprintf(stderr, "Framebytes has changed! Can't deal with this, aborting\n");
break;
}
framesread++;
//check for missing frames
while(framemjd != nextmjd || framesecond != nextsecond || framenumber != nextnumber) {
if(VERBOSE)
fprintf(stderr, "Missed a packet! We were expecting MJD %d, sec %d, frame %d, and the next one came in MJD %d, sec %d, frame %d\n",
nextmjd, nextsecond, nextnumber, framemjd, framesecond, framenumber);
//write out an invalid frame for the next frame
overwritenumber = nextnumber + offsetnumber;
overwritesecond = nextsecond + offsetsecond;
overwritemjd = nextmjd + offsetmjd;
while (overwritenumber < 0) {
overwritenumber += framespersecond;
overwritesecond++;
}
while (overwritenumber >= framespersecond) {
overwritenumber -= framespersecond;
overwritesecond--;
}
while (overwritesecond < 0) {
overwritesecond += 86400;
overwritemjd++;
}
while (overwritesecond >= 86400) {
overwritesecond -= 86400;
overwritemjd--;
}
setVDIFFrameMJD(header, overwritemjd);
setVDIFFrameSecond(header, overwritesecond);
setVDIFFrameNumber(header, overwritenumber);
setVDIFFrameInvalid(header, 1);
readbytes = fwrite(buffer, 1, framebytes, output); //write out the VDIF packet
if(readbytes < framebytes) {
fprintf(stderr, "Problem writing %lldth frame - only wrote %d bytes\n", framesread, readbytes);
break;
}
frameswrote++;
nextnumber++;
if(nextnumber == framespersecond) {
nextnumber = 0;
nextsecond++;
if(nextsecond == 86400) {
nextsecond = 0;
nextmjd++;
}
}
}
overwritenumber = framenumber + offsetnumber;
overwritesecond = framesecond + offsetsecond;
overwritemjd = framemjd + offsetmjd;
while (overwritenumber < 0) {
overwritenumber += framespersecond;
overwritesecond++;
}
while (overwritenumber >= framespersecond) {
overwritenumber -= framespersecond;
overwritesecond--;
}
while (overwritesecond < 0) {
overwritesecond += 86400;
overwritemjd++;
}
while (overwritesecond >= 86400) {
overwritesecond -= 86400;
overwritemjd--;
}
header = (vdif_header*)buffer;
setVDIFFrameMJD(header, overwritemjd);
setVDIFFrameSecond(header, overwritesecond);
setVDIFFrameNumber(header, overwritenumber);
if(FORCE_VALID)
frameinvalid = 0;
setVDIFFrameInvalid(header, frameinvalid);
readbytes = fwrite(buffer, 1, framebytes, output); //write out the VDIF packet
if(readbytes < framebytes) {
fprintf(stderr, "Problem writing %lldth frame - only wrote %d bytes\n", framesread, readbytes);
break;
}
frameswrote++;
nextnumber++;
if(nextnumber == framespersecond) {
nextnumber = 0;
nextsecond++;
if(nextsecond == 86400) {
nextsecond = 0;
nextmjd++;
}
}
}
printf("Read %lld and wrote %lld frames\n", framesread, frameswrote);
fclose(input);
fclose(output);
return EXIT_SUCCESS;
}
int main(int argc, char **argv) {
int c=0;
int check = 0;
int fix = 0;
int verbose = 0;
int test = 0;
char *infile = NULL;
FILE *input = stdin;
FILE *output = stdout;
int rate = 10000;
if (argc > 1) {
while ((c = getopt(argc, argv, "i:ctfr:v")) != -1) {
switch(c) {
case 'c':
check = 1;
break;
case 'f':
fix = 1;
break;
case 'i':
infile = strdup(optarg);
input = fopen(infile,"r");
break;
case 'r':
rate = atoi(optarg);
break;
case 't':
test = 1;
break;
case 'v':
verbose = 1;
break;
default:
usage();
break;
}
}
}
else {
usage();
}
vdif_header vhdr;
vdif_header last_vhdr;
unsigned int frame = 0;
int finished = 0;
unsigned int rtn = 0;
int got_first = 0;
char *buffer = NULL;
char *pattern = NULL;
int second = 0;
int last_second = 0;
int last_frame=0;
double MJD = 0.0;
unsigned long frame_count = 0;
int failed = 0;
int diff_frame=0;
int diff_seconds=0;
if (check || fix || test) {
while (!finished) {
rtn = fread(&vhdr,32,1,input);
if (feof(input) || rtn != 1) {
if (verbose)
fprintf(stderr,"chk_vdif finished:\n");
finished = 1;
continue;
}
else {
frame_count = frame_count + 1;
}
if (got_first == 0) {
if (check && verbose)
fprintf(stdout,"## Frame length: %d",vhdr.framelength8*8);
buffer = malloc(vhdr.framelength8*8);
assert(buffer);
if (fix) {
// initialise to a mean zero 2bit offset binary pattern
pattern = malloc(vhdr.framelength8*8);
assert(pattern);
uint16_t mean_zero = 0xAA55;
size_t count = 0;
while (count < vhdr.framelength8*8) {
memcpy(pattern+count,&mean_zero,2);
count = count+2;
}
}
got_first = 1;
}
else {
frame = getVDIFFrameNumber(&vhdr);
if (check && verbose)
fprintf(stdout,"Frame: %d\n",frame);
second = getVDIFFrameSecond(&vhdr);
if (check && verbose)
fprintf(stdout,"Second: %d\n", second) ;
MJD = getVDIFDMJD(&vhdr,10000);
if (check && verbose)
fprintf(stdout,"MJD %f\n",MJD);
if (frame_count > 2) {
diff_frame = frame-last_frame;
if (diff_frame > 1) {
if (check)
fprintf(stderr,"Skipped a frame between %d and %d\n",last_frame,frame);
failed = 1;
}
diff_seconds = second-last_second;
if (diff_seconds > 1) {
if (check)
fprintf(stderr,"Skipped a second between %d and %d\n",last_second,second);
if (fix) {
while (diff_seconds > 1) {
nextVDIFHeader(&last_vhdr,rate);
fwrite(&last_vhdr,VDIF_HEADER_BYTES,1,stdout);
// repeat the last buffer
fwrite(&buffer,(vhdr.framelength8*8-VDIF_HEADER_BYTES),1,output);
last_second = getVDIFFrameSecond(&last_vhdr);
diff_seconds = second-last_second;
}
}
if (fix == 0)
failed = 1;
}
}
}
if (failed == 1) {
fprintf(stdout,"failed\n");
finished = 1;
continue;
}
rtn = fread(buffer,(vhdr.framelength8*8-VDIF_HEADER_BYTES),1,input);
if (feof(input) || rtn != 1) {
if (verbose)
fprintf(stderr,"chk_vdif finished:\n");
finished = 1;
continue;
}
last_frame = frame;
last_second = second;
last_vhdr = vhdr;
if (fix) {
fwrite(&vhdr,VDIF_HEADER_BYTES,1,output);
fwrite(buffer,(vhdr.framelength8*8-VDIF_HEADER_BYTES),1,output);
}
}
}
free(buffer);
if (test && failed == 0) {
fprintf(stdout,"passed\n");
}
if (infile) {
fclose(input);
}
}
void summarize(const char *fileName)
{
FILE *in;
Mark6Header H;
char *buffer;
char *data;
int i;
Mark6BlockHeader_ver2 *h;
int s;
vdif_header *v1, *v2;
in = fopen(fileName, "r");
if(in == 0)
{
fprintf(stderr, "Cannot open %s for read\n", fileName);
return;
}
fread(&H, sizeof(H), 1, in);
printf("File: %s\n", fileName);
printMark6Header(&H);
if(H.version < 2)
{
return;
}
s = sizeof(Mark6BlockHeader_ver2);
buffer = (char *)malloc(H.block_size);
h = (Mark6BlockHeader_ver2 *)buffer;
data = buffer + s;
v1 = (vdif_header *)data;
for(i = 0; ; ++i)
{
int v, j, n;
int nSize;
nSize = 0;
v = fread(buffer, s, 1, in);
if(v < 1)
{
break;
}
n = (h->wb_size-s)/H.packet_size;
v = fread(buffer+s, 1, h->wb_size-s, in);
if(v < s)
{
printf("Early EOF: only %d bytes read. %d bytes expected.\n", v, h->wb_size-s);
}
for(j = 0; j < n; ++j)
{
v2 = (vdif_header *)(data + j*H.packet_size);
if(getVDIFFrameBytes(v2) == H.packet_size)
{
++nSize;
}
}
printf("%d %d %d/%d %d:%05d:%d - %d:%05d:%d\n", i, h->blocknum, nSize, n,
getVDIFFrameEpochSecOffset(v1), getVDIFFrameNumber(v1), getVDIFThreadID(v1),
getVDIFFrameEpochSecOffset(v2), getVDIFFrameNumber(v2), getVDIFThreadID(v2));
}
free(buffer);
}
int main(int argc, char** argv)
{
ipcio_t data_block = IPCIO_INIT;
key_t key = DADA_DEFAULT_BLOCK_KEY;
int status = 0;
char eventfile[128];
FILE *infd, *evfd;
//get this many bytes at a time from data socket
int BUFSIZE = UDP_HDR_SIZE + VDIF_PKT_SIZE;
char buf[BUFSIZE];
char dev[16] = ETHDEV;
char hostname[MAXHOSTNAME];
gethostname(hostname,MAXHOSTNAME);
char *starthost = strstr(hostname, "difx");
int nbytes = 0, state = STATE_STOPPED;
uint64_t port = WRITER_SERVICE_PORT;
//int cmd = CMD_NONE;
char cmd = CMD_NONE;
int ii, arg, maxsock = 0;
fd_set readfds;
struct timeval tv; //timeout for select()--set it to zero
tv.tv_sec = 0;
tv.tv_usec = 0;
Connection c;
c.sockoptval = 1; //release port immediately after closing connection
Connection raw;
raw.alen = sizeof(raw.rem_addr);
raw.sockoptval = 32*1024*1024; //size of data socket internal buffer
// For gathering stats on frame skips
long framenumtmp, framenum[2], framediff;
framenum[0] = -1;
framenum[1] = -1;
int threadid = -1;
FILE* skiplogfd;
char skiplogfile[128];
time_t currt;
char currt_string[128];
struct tm *tmpt;
while ((arg = getopt(argc, argv, "hk:p:e:")) != -1) {
switch(arg) {
case 'h':
usage ();
return 0;
case 'k':
if (sscanf (optarg, "%x", &key) != 1) {
fprintf (stderr, "writer: could not parse key from %s\n", optarg);
return -1;
}
break;
case 'p':
if (sscanf (optarg, "%"PRIu64"", &port) != 1) {
fprintf (stderr, "writer: could not parse port from %s\n", optarg);
return -1;
}
break;
case 'e':
if (sscanf (optarg, "%s", dev) != 1) {
fprintf (stderr, "writer: could not parse ethernet device from %s\n", optarg);
return -1;
}
break;
}
}
sprintf(skiplogfile,"%s%s%s%s","skiplog-",starthost,dev,".txt");
if((skiplogfd = fopen(skiplogfile, "w")) == NULL) {
fprintf(stderr,"Writer: Could not open skiplog file %s for writing.\n",skiplogfile);
exit(1);
}
//Try to connect to existing data block
if(ipcio_connect(&data_block,key) < 0)
exit(1);
fprintf(stderr,"after ipcio_connect\n");
//create listening socket
if(serve(port, &c) < 0) {
fprintf(stderr,"Writer: Failed to create listening socket.\n");
exit(1);
}
maxsock = c.rqst;
//Open raw data stream socket
raw.svc = openRawSocket(dev,0);
if(raw.svc < 0) {
fprintf(stderr, "Cannot open raw socket on %s. Error code %d\n", dev, raw.svc);
exit(1);
}
setsockopt(raw.svc, SOL_SOCKET, SO_RCVBUF, (char *)&(raw.sockoptval), sizeof(raw.sockoptval));
if(raw.svc > maxsock)
maxsock = raw.svc;
//ii = 0;
while(1) {
//ii++;
//printf("ii: %d state: %d cmd: %d\n",ii,state,cmd);
if(state == STATE_STOPPED) {
if(cmd == CMD_NONE)
cmd = wait_for_cmd(&c);
if(cmd == CMD_START) {
state = STATE_STARTED;
if(ipcio_open(&data_block,'W') < 0)
exit(1);
fprintf(stderr,"after ipcio_open, W\n");
cmd = CMD_NONE;
}
else if(cmd == CMD_EVENT) {
fprintf(stderr, "Writer: ignored CMD_EVENT in STATE_STOPPED.\n");
cmd = CMD_NONE;
}
else if(cmd == CMD_STOP) {
fprintf(stderr, "Writer: ignored CMD_STOP in STATE_STOPPED.\n");
cmd = CMD_NONE;
}
else if(cmd == CMD_QUIT) {
shutdown(c.rqst,2);
return 0;
}
}
//if state is STARTED, poll the command listening socket and the VDIF raw data socket
if(state == STATE_STARTED) {
FD_ZERO(&readfds);
FD_SET(c.rqst, &readfds);
FD_SET(raw.svc, &readfds);
select(maxsock+1,&readfds,NULL,NULL,&tv);
//if input is waiting on listening socket, read it
if(FD_ISSET(c.rqst,&readfds)) {
cmd = wait_for_cmd(&c);
}
if(cmd == CMD_EVENT) {
//close data block
if(ipcio_close(&data_block) < 0)
exit(1);
fprintf(stderr,"after ipcio_close\n");
//dump DB to file
currt = time(NULL);
tmpt = localtime(&currt);
strftime(currt_string,sizeof(currt_string), "%Y%m%d_%H%M%S", tmpt);
*(currt_string+15) = 0;
sprintf(eventfile,"%s/%s%s_%s_ev.out",EVENTDIR,starthost,dev,currt_string);
if((evfd = fopen(eventfile,"w")) == NULL) {
fprintf(stderr,"Writer: Could not open file %s for writing.\n",eventfile);
}
else {
event_to_file(&data_block,evfd);
fclose(evfd);
}
//Get pending commands, resume writing to ring buffer if there are none
FD_ZERO(&readfds);
FD_SET(c.rqst, &readfds);
FD_SET(raw.svc, &readfds);
select(maxsock+1,&readfds,NULL,NULL,&tv);
state = STATE_STOPPED;
if(FD_ISSET(c.rqst,&readfds)) {
cmd = wait_for_cmd(&c);
fprintf(stderr,"Writer: flushed out command socket after event_to_file.\n");
}
else
cmd = CMD_START;
continue;
}
//if command is stop, change state to STOPPED, close data block
else if(cmd == CMD_STOP) {
state = STATE_STOPPED;
if(ipcio_close(&data_block) < 0)
exit(1);
fprintf(stderr,"after ipcio_close\n");
cmd = CMD_NONE;
continue;
}
//if command is quit, close data block, shutdown listening socket, return
else if(cmd == CMD_QUIT) {
if(ipcio_close(&data_block) < 0)
exit(1);
fprintf(stderr,"after ipcio_close\n");
shutdown(c.rqst,2);
return 0;
}
else if(cmd == CMD_START) {
fprintf(stderr,"Writer: ignored cmd start (already started).\n");
cmd = CMD_NONE;
}
//read a packet from the data socket and write it to the ring buffer
if(FD_ISSET(raw.svc,&readfds)) {
nbytes = recvfrom(raw.svc, buf, BUFSIZE, 0, (struct sockaddr *)&(raw.rem_addr), &(raw.alen));
if(nbytes == BUFSIZE) {
//fwrite(buf + trim, recvlen-trim, 1, out);
status = ipcio_write(&data_block,buf+UDP_HDR_SIZE,VDIF_PKT_SIZE);
//fprintf(stderr,"ipcio_write: %d bytes\n",status);
//print VDIF frame header
//printVDIFHeader((vdif_header *)(buf+UDP_HDR_SIZE), VDIFHeaderPrintLevelLong);
//printVDIFHeader((vdif_header *)(buf+UDP_HDR_SIZE), VDIFHeaderPrintLevelColumns);
//printVDIFHeader((vdif_header *)(buf+UDP_HDR_SIZE), VDIFHeaderPrintLevelShort);
//fprintf(stderr,"VDIFFrame MJD: %d Number: %d\n",getVDIFFrameMJD((vdif_header *)(buf+UDP_HDR_SIZE)), getVDIFFrameNumber((vdif_header *)(buf+UDP_HDR_SIZE)));
framenumtmp = getVDIFFrameNumber((vdif_header *)(buf+UDP_HDR_SIZE));
threadid = getVDIFThreadID((vdif_header *)(buf+UDP_HDR_SIZE));
framediff = framenumtmp - framenum[threadid];
if(framediff != -MAXFRAMENUM && framenum[threadid] != -1 && framediff != 1) {
//if(framediff != -25599 && abs(framediff) > 1 && framenum[threadid] != -1) {
currt = time(NULL);
tmpt = localtime(&currt);
strftime(currt_string,sizeof(currt_string), "%Y-%m-%d %H:%M:%S", tmpt);
fprintf(skiplogfd,"%s FRAME SKIP FROM %d to %d (THREAD %d)\n",currt_string,framenum[threadid],framenumtmp,threadid);
fflush(skiplogfd);
}
else if(framenum[threadid] == -1) {
fprintf(skiplogfd,"Writer: Thread %d First frame: %d\n",threadid,framenumtmp);
fflush(skiplogfd);
}
framenum[threadid] = framenumtmp;
}
else if(nbytes <= 0) {
fprintf(stderr,"Raw socket read failed: %d\n.", nbytes);
}
/*
else
fprintf(stderr,"Received packet size: %d, ignoring.\n", nbytes);
*/
}
//XXX: check for overflow of the data block here?
//nbytes = fread(buf,1,BUFSIZE,infd);
//fprintf(stderr,"fread: %d bytes\n",nbytes);
//status = ipcio_write(&data_block,buf,nbytes);
//fprintf(stderr,"ipcio_write: %d bytes\n",status);
//sleep(2);
}
}
//fclose(infd);
return 0;
}
