int main(int argc, char **argv) { const int MaxFrameSize = 16*MAX_VDIF_FRAME_BYTES; char buffer[MaxFrameSize]; FILE * input; int readbytes, framebytes, framemjd, framesecond, framenumber, frameinvalid, datambps, framespersecond; long long framesread; vdif_header *header; if(argc != 3) { usage(); return EXIT_FAILURE; } input = fopen(argv[1], "r"); if(input == NULL) { fprintf(stderr, "Cannot open input file %s\n", argv[1]); exit(EXIT_FAILURE); } datambps = atoi(argv[2]); readbytes = fread(buffer, 1, MaxFrameSize, input); //read the VDIF header header = (vdif_header*)buffer; framebytes = getVDIFFrameBytes(header); if(framebytes > MaxFrameSize) { fprintf(stderr, "Cannot read frame with %d bytes > max (%d), formal max %d\n", framebytes, MaxFrameSize, MAX_VDIF_FRAME_BYTES); exit(EXIT_FAILURE); } framemjd = getVDIFFrameMJD(header); framesecond = getVDIFFrameSecond(header); framenumber = getVDIFFrameNumber(header); frameinvalid = getVDIFFrameInvalid(header); framespersecond = (int)((((long long)datambps)*1000000)/(8*(framebytes-VDIF_HEADER_BYTES))); printf("Frames per second is %d\n", framespersecond); rewind(input); //go back to the start framesread = 0; while(!feof(input)) { readbytes = fread(buffer, 1, framebytes, input); //read the whole VDIF packet if (readbytes < framebytes) { fprintf(stderr, "Header read failed - probably at end of file.\n"); break; } header = (vdif_header*)buffer; framemjd = getVDIFFrameMJD(header); framesecond = getVDIFFrameSecond(header); framenumber = getVDIFFrameNumber(header); frameinvalid = getVDIFFrameInvalid(header); printf("MJD is %d, second is %d, framenumber is %d, frameinvalid is %d\n", framemjd, framesecond, framenumber, frameinvalid); printf("Threadid is %d, stationid is %d, numbits is %d\n", getVDIFThreadID(header), getVDIFStationID(header), getVDIFBitsPerSample(header)); if(getVDIFFrameBytes(header) != framebytes) { fprintf(stderr, "Framebytes has changed! Can't deal with this, aborting\n"); break; } framesread++; } printf("Read %lld frames\n", framesread); fclose(input); return EXIT_SUCCESS; }
int main(int argc, char **argv) { int VERBOSE = 0; int FORCE_VALID = 1; char buffer[MAX_VDIF_FRAME_BYTES]; FILE * input; FILE * output; int readbytes, framebytes, framemjd, framesecond, framenumber, frameinvalid, datambps, framespersecond; int nextmjd, nextsecond, nextnumber; int overwritemjd, overwritesecond, overwritenumber; int offsetmjd=0, offsetsecond=0, offsetnumber=0; long long framesread, frameswrote; vdif_header *header; if(argc != 4 && argc != 5) { usage(); return EXIT_FAILURE; } overwritemjd = 0; overwritesecond = 0; overwritenumber = 0; if(argc == 5) { overwritemjd = atoi(argv[4]); overwritesecond = (int)(86400.0*(atof(argv[4]) - overwritemjd)); } input = fopen(argv[1], "r"); if(input == NULL) { fprintf(stderr, "Cannot open input file %s\n", argv[1]); exit(EXIT_FAILURE); } output = fopen(argv[2], "w"); if(output == NULL) { fprintf(stderr, "Cannot open output file %s\n", argv[2]); exit(EXIT_FAILURE); } if(FORCE_VALID) { printf("Forcing all read frames to be VALID! (Temp workaround for invalid tagged EVLA VDIF data)\n"); } datambps = atoi(argv[3]); readbytes = fread(buffer, 1, VDIF_HEADER_BYTES, input); //read the VDIF header header = (vdif_header*)buffer; framebytes = getVDIFFrameBytes(header); if(framebytes > MAX_VDIF_FRAME_BYTES) { fprintf(stderr, "Cannot read frame with %d bytes > max (%d)\n", framebytes, MAX_VDIF_FRAME_BYTES); exit(EXIT_FAILURE); } nextmjd = getVDIFFrameMJD(header); nextsecond = getVDIFFrameSecond(header); nextnumber = getVDIFFrameNumber(header); if(overwritemjd > 0) { offsetmjd = overwritemjd - nextmjd; offsetsecond = overwritesecond - nextsecond; offsetnumber = overwritenumber - nextnumber; } framespersecond = (int)((((long long)datambps)*1000000)/(8*(framebytes-VDIF_HEADER_BYTES))); printf("Frames per second is %d\n", framespersecond); rewind(input); //go back to the start framesread = 0; frameswrote = 0; while(!feof(input)) { readbytes = fread(buffer, 1, framebytes, input); //read the whole VDIF packet if (readbytes < framebytes) { fprintf(stderr, "Header read failed - probably at end of file.\n"); break; } header = (vdif_header*)buffer; framemjd = getVDIFFrameMJD(header); framesecond = getVDIFFrameSecond(header); framenumber = getVDIFFrameNumber(header); frameinvalid = getVDIFFrameInvalid(header); if(getVDIFFrameBytes(header) != framebytes) { fprintf(stderr, "Framebytes has changed! Can't deal with this, aborting\n"); break; } framesread++; //check for missing frames while(framemjd != nextmjd || framesecond != nextsecond || framenumber != nextnumber) { if(VERBOSE) fprintf(stderr, "Missed a packet! We were expecting MJD %d, sec %d, frame %d, and the next one came in MJD %d, sec %d, frame %d\n", nextmjd, nextsecond, nextnumber, framemjd, framesecond, framenumber); //write out an invalid frame for the next frame overwritenumber = nextnumber + offsetnumber; overwritesecond = nextsecond + offsetsecond; overwritemjd = nextmjd + offsetmjd; while (overwritenumber < 0) { overwritenumber += framespersecond; overwritesecond++; } while (overwritenumber >= framespersecond) { overwritenumber -= framespersecond; overwritesecond--; } while (overwritesecond < 0) { overwritesecond += 86400; overwritemjd++; } while (overwritesecond >= 86400) { overwritesecond -= 86400; overwritemjd--; } setVDIFFrameMJD(header, overwritemjd); setVDIFFrameSecond(header, overwritesecond); setVDIFFrameNumber(header, overwritenumber); setVDIFFrameInvalid(header, 1); readbytes = fwrite(buffer, 1, framebytes, output); //write out the VDIF packet if(readbytes < framebytes) { fprintf(stderr, "Problem writing %lldth frame - only wrote %d bytes\n", framesread, readbytes); break; } frameswrote++; nextnumber++; if(nextnumber == framespersecond) { nextnumber = 0; nextsecond++; if(nextsecond == 86400) { nextsecond = 0; nextmjd++; } } } overwritenumber = framenumber + offsetnumber; overwritesecond = framesecond + offsetsecond; overwritemjd = framemjd + offsetmjd; while (overwritenumber < 0) { overwritenumber += framespersecond; overwritesecond++; } while (overwritenumber >= framespersecond) { overwritenumber -= framespersecond; overwritesecond--; } while (overwritesecond < 0) { overwritesecond += 86400; overwritemjd++; } while (overwritesecond >= 86400) { overwritesecond -= 86400; overwritemjd--; } header = (vdif_header*)buffer; setVDIFFrameMJD(header, overwritemjd); setVDIFFrameSecond(header, overwritesecond); setVDIFFrameNumber(header, overwritenumber); if(FORCE_VALID) frameinvalid = 0; setVDIFFrameInvalid(header, frameinvalid); readbytes = fwrite(buffer, 1, framebytes, output); //write out the VDIF packet if(readbytes < framebytes) { fprintf(stderr, "Problem writing %lldth frame - only wrote %d bytes\n", framesread, readbytes); break; } frameswrote++; nextnumber++; if(nextnumber == framespersecond) { nextnumber = 0; nextsecond++; if(nextsecond == 86400) { nextsecond = 0; nextmjd++; } } } printf("Read %lld and wrote %lld frames\n", framesread, frameswrote); fclose(input); fclose(output); return EXIT_SUCCESS; }
int main(int argc, char **argv) { int c=0; int check = 0; int fix = 0; int verbose = 0; int test = 0; char *infile = NULL; FILE *input = stdin; FILE *output = stdout; int rate = 10000; if (argc > 1) { while ((c = getopt(argc, argv, "i:ctfr:v")) != -1) { switch(c) { case 'c': check = 1; break; case 'f': fix = 1; break; case 'i': infile = strdup(optarg); input = fopen(infile,"r"); break; case 'r': rate = atoi(optarg); break; case 't': test = 1; break; case 'v': verbose = 1; break; default: usage(); break; } } } else { usage(); } vdif_header vhdr; vdif_header last_vhdr; unsigned int frame = 0; int finished = 0; unsigned int rtn = 0; int got_first = 0; char *buffer = NULL; char *pattern = NULL; int second = 0; int last_second = 0; int last_frame=0; double MJD = 0.0; unsigned long frame_count = 0; int failed = 0; int diff_frame=0; int diff_seconds=0; if (check || fix || test) { while (!finished) { rtn = fread(&vhdr,32,1,input); if (feof(input) || rtn != 1) { if (verbose) fprintf(stderr,"chk_vdif finished:\n"); finished = 1; continue; } else { frame_count = frame_count + 1; } if (got_first == 0) { if (check && verbose) fprintf(stdout,"## Frame length: %d",vhdr.framelength8*8); buffer = malloc(vhdr.framelength8*8); assert(buffer); if (fix) { // initialise to a mean zero 2bit offset binary pattern pattern = malloc(vhdr.framelength8*8); assert(pattern); uint16_t mean_zero = 0xAA55; size_t count = 0; while (count < vhdr.framelength8*8) { memcpy(pattern+count,&mean_zero,2); count = count+2; } } got_first = 1; } else { frame = getVDIFFrameNumber(&vhdr); if (check && verbose) fprintf(stdout,"Frame: %d\n",frame); second = getVDIFFrameSecond(&vhdr); if (check && verbose) fprintf(stdout,"Second: %d\n", second) ; MJD = getVDIFDMJD(&vhdr,10000); if (check && verbose) fprintf(stdout,"MJD %f\n",MJD); if (frame_count > 2) { diff_frame = frame-last_frame; if (diff_frame > 1) { if (check) fprintf(stderr,"Skipped a frame between %d and %d\n",last_frame,frame); failed = 1; } diff_seconds = second-last_second; if (diff_seconds > 1) { if (check) fprintf(stderr,"Skipped a second between %d and %d\n",last_second,second); if (fix) { while (diff_seconds > 1) { nextVDIFHeader(&last_vhdr,rate); fwrite(&last_vhdr,VDIF_HEADER_BYTES,1,stdout); // repeat the last buffer fwrite(&buffer,(vhdr.framelength8*8-VDIF_HEADER_BYTES),1,output); last_second = getVDIFFrameSecond(&last_vhdr); diff_seconds = second-last_second; } } if (fix == 0) failed = 1; } } } if (failed == 1) { fprintf(stdout,"failed\n"); finished = 1; continue; } rtn = fread(buffer,(vhdr.framelength8*8-VDIF_HEADER_BYTES),1,input); if (feof(input) || rtn != 1) { if (verbose) fprintf(stderr,"chk_vdif finished:\n"); finished = 1; continue; } last_frame = frame; last_second = second; last_vhdr = vhdr; if (fix) { fwrite(&vhdr,VDIF_HEADER_BYTES,1,output); fwrite(buffer,(vhdr.framelength8*8-VDIF_HEADER_BYTES),1,output); } } } free(buffer); if (test && failed == 0) { fprintf(stdout,"passed\n"); } if (infile) { fclose(input); } }
void summarize(const char *fileName) { FILE *in; Mark6Header H; char *buffer; char *data; int i; Mark6BlockHeader_ver2 *h; int s; vdif_header *v1, *v2; in = fopen(fileName, "r"); if(in == 0) { fprintf(stderr, "Cannot open %s for read\n", fileName); return; } fread(&H, sizeof(H), 1, in); printf("File: %s\n", fileName); printMark6Header(&H); if(H.version < 2) { return; } s = sizeof(Mark6BlockHeader_ver2); buffer = (char *)malloc(H.block_size); h = (Mark6BlockHeader_ver2 *)buffer; data = buffer + s; v1 = (vdif_header *)data; for(i = 0; ; ++i) { int v, j, n; int nSize; nSize = 0; v = fread(buffer, s, 1, in); if(v < 1) { break; } n = (h->wb_size-s)/H.packet_size; v = fread(buffer+s, 1, h->wb_size-s, in); if(v < s) { printf("Early EOF: only %d bytes read. %d bytes expected.\n", v, h->wb_size-s); } for(j = 0; j < n; ++j) { v2 = (vdif_header *)(data + j*H.packet_size); if(getVDIFFrameBytes(v2) == H.packet_size) { ++nSize; } } printf("%d %d %d/%d %d:%05d:%d - %d:%05d:%d\n", i, h->blocknum, nSize, n, getVDIFFrameEpochSecOffset(v1), getVDIFFrameNumber(v1), getVDIFThreadID(v1), getVDIFFrameEpochSecOffset(v2), getVDIFFrameNumber(v2), getVDIFThreadID(v2)); } free(buffer); }
int main(int argc, char** argv) { ipcio_t data_block = IPCIO_INIT; key_t key = DADA_DEFAULT_BLOCK_KEY; int status = 0; char eventfile[128]; FILE *infd, *evfd; //get this many bytes at a time from data socket int BUFSIZE = UDP_HDR_SIZE + VDIF_PKT_SIZE; char buf[BUFSIZE]; char dev[16] = ETHDEV; char hostname[MAXHOSTNAME]; gethostname(hostname,MAXHOSTNAME); char *starthost = strstr(hostname, "difx"); int nbytes = 0, state = STATE_STOPPED; uint64_t port = WRITER_SERVICE_PORT; //int cmd = CMD_NONE; char cmd = CMD_NONE; int ii, arg, maxsock = 0; fd_set readfds; struct timeval tv; //timeout for select()--set it to zero tv.tv_sec = 0; tv.tv_usec = 0; Connection c; c.sockoptval = 1; //release port immediately after closing connection Connection raw; raw.alen = sizeof(raw.rem_addr); raw.sockoptval = 32*1024*1024; //size of data socket internal buffer // For gathering stats on frame skips long framenumtmp, framenum[2], framediff; framenum[0] = -1; framenum[1] = -1; int threadid = -1; FILE* skiplogfd; char skiplogfile[128]; time_t currt; char currt_string[128]; struct tm *tmpt; while ((arg = getopt(argc, argv, "hk:p:e:")) != -1) { switch(arg) { case 'h': usage (); return 0; case 'k': if (sscanf (optarg, "%x", &key) != 1) { fprintf (stderr, "writer: could not parse key from %s\n", optarg); return -1; } break; case 'p': if (sscanf (optarg, "%"PRIu64"", &port) != 1) { fprintf (stderr, "writer: could not parse port from %s\n", optarg); return -1; } break; case 'e': if (sscanf (optarg, "%s", dev) != 1) { fprintf (stderr, "writer: could not parse ethernet device from %s\n", optarg); return -1; } break; } } sprintf(skiplogfile,"%s%s%s%s","skiplog-",starthost,dev,".txt"); if((skiplogfd = fopen(skiplogfile, "w")) == NULL) { fprintf(stderr,"Writer: Could not open skiplog file %s for writing.\n",skiplogfile); exit(1); } //Try to connect to existing data block if(ipcio_connect(&data_block,key) < 0) exit(1); fprintf(stderr,"after ipcio_connect\n"); //create listening socket if(serve(port, &c) < 0) { fprintf(stderr,"Writer: Failed to create listening socket.\n"); exit(1); } maxsock = c.rqst; //Open raw data stream socket raw.svc = openRawSocket(dev,0); if(raw.svc < 0) { fprintf(stderr, "Cannot open raw socket on %s. Error code %d\n", dev, raw.svc); exit(1); } setsockopt(raw.svc, SOL_SOCKET, SO_RCVBUF, (char *)&(raw.sockoptval), sizeof(raw.sockoptval)); if(raw.svc > maxsock) maxsock = raw.svc; //ii = 0; while(1) { //ii++; //printf("ii: %d state: %d cmd: %d\n",ii,state,cmd); if(state == STATE_STOPPED) { if(cmd == CMD_NONE) cmd = wait_for_cmd(&c); if(cmd == CMD_START) { state = STATE_STARTED; if(ipcio_open(&data_block,'W') < 0) exit(1); fprintf(stderr,"after ipcio_open, W\n"); cmd = CMD_NONE; } else if(cmd == CMD_EVENT) { fprintf(stderr, "Writer: ignored CMD_EVENT in STATE_STOPPED.\n"); cmd = CMD_NONE; } else if(cmd == CMD_STOP) { fprintf(stderr, "Writer: ignored CMD_STOP in STATE_STOPPED.\n"); cmd = CMD_NONE; } else if(cmd == CMD_QUIT) { shutdown(c.rqst,2); return 0; } } //if state is STARTED, poll the command listening socket and the VDIF raw data socket if(state == STATE_STARTED) { FD_ZERO(&readfds); FD_SET(c.rqst, &readfds); FD_SET(raw.svc, &readfds); select(maxsock+1,&readfds,NULL,NULL,&tv); //if input is waiting on listening socket, read it if(FD_ISSET(c.rqst,&readfds)) { cmd = wait_for_cmd(&c); } if(cmd == CMD_EVENT) { //close data block if(ipcio_close(&data_block) < 0) exit(1); fprintf(stderr,"after ipcio_close\n"); //dump DB to file currt = time(NULL); tmpt = localtime(&currt); strftime(currt_string,sizeof(currt_string), "%Y%m%d_%H%M%S", tmpt); *(currt_string+15) = 0; sprintf(eventfile,"%s/%s%s_%s_ev.out",EVENTDIR,starthost,dev,currt_string); if((evfd = fopen(eventfile,"w")) == NULL) { fprintf(stderr,"Writer: Could not open file %s for writing.\n",eventfile); } else { event_to_file(&data_block,evfd); fclose(evfd); } //Get pending commands, resume writing to ring buffer if there are none FD_ZERO(&readfds); FD_SET(c.rqst, &readfds); FD_SET(raw.svc, &readfds); select(maxsock+1,&readfds,NULL,NULL,&tv); state = STATE_STOPPED; if(FD_ISSET(c.rqst,&readfds)) { cmd = wait_for_cmd(&c); fprintf(stderr,"Writer: flushed out command socket after event_to_file.\n"); } else cmd = CMD_START; continue; } //if command is stop, change state to STOPPED, close data block else if(cmd == CMD_STOP) { state = STATE_STOPPED; if(ipcio_close(&data_block) < 0) exit(1); fprintf(stderr,"after ipcio_close\n"); cmd = CMD_NONE; continue; } //if command is quit, close data block, shutdown listening socket, return else if(cmd == CMD_QUIT) { if(ipcio_close(&data_block) < 0) exit(1); fprintf(stderr,"after ipcio_close\n"); shutdown(c.rqst,2); return 0; } else if(cmd == CMD_START) { fprintf(stderr,"Writer: ignored cmd start (already started).\n"); cmd = CMD_NONE; } //read a packet from the data socket and write it to the ring buffer if(FD_ISSET(raw.svc,&readfds)) { nbytes = recvfrom(raw.svc, buf, BUFSIZE, 0, (struct sockaddr *)&(raw.rem_addr), &(raw.alen)); if(nbytes == BUFSIZE) { //fwrite(buf + trim, recvlen-trim, 1, out); status = ipcio_write(&data_block,buf+UDP_HDR_SIZE,VDIF_PKT_SIZE); //fprintf(stderr,"ipcio_write: %d bytes\n",status); //print VDIF frame header //printVDIFHeader((vdif_header *)(buf+UDP_HDR_SIZE), VDIFHeaderPrintLevelLong); //printVDIFHeader((vdif_header *)(buf+UDP_HDR_SIZE), VDIFHeaderPrintLevelColumns); //printVDIFHeader((vdif_header *)(buf+UDP_HDR_SIZE), VDIFHeaderPrintLevelShort); //fprintf(stderr,"VDIFFrame MJD: %d Number: %d\n",getVDIFFrameMJD((vdif_header *)(buf+UDP_HDR_SIZE)), getVDIFFrameNumber((vdif_header *)(buf+UDP_HDR_SIZE))); framenumtmp = getVDIFFrameNumber((vdif_header *)(buf+UDP_HDR_SIZE)); threadid = getVDIFThreadID((vdif_header *)(buf+UDP_HDR_SIZE)); framediff = framenumtmp - framenum[threadid]; if(framediff != -MAXFRAMENUM && framenum[threadid] != -1 && framediff != 1) { //if(framediff != -25599 && abs(framediff) > 1 && framenum[threadid] != -1) { currt = time(NULL); tmpt = localtime(&currt); strftime(currt_string,sizeof(currt_string), "%Y-%m-%d %H:%M:%S", tmpt); fprintf(skiplogfd,"%s FRAME SKIP FROM %d to %d (THREAD %d)\n",currt_string,framenum[threadid],framenumtmp,threadid); fflush(skiplogfd); } else if(framenum[threadid] == -1) { fprintf(skiplogfd,"Writer: Thread %d First frame: %d\n",threadid,framenumtmp); fflush(skiplogfd); } framenum[threadid] = framenumtmp; } else if(nbytes <= 0) { fprintf(stderr,"Raw socket read failed: %d\n.", nbytes); } /* else fprintf(stderr,"Received packet size: %d, ignoring.\n", nbytes); */ } //XXX: check for overflow of the data block here? //nbytes = fread(buf,1,BUFSIZE,infd); //fprintf(stderr,"fread: %d bytes\n",nbytes); //status = ipcio_write(&data_block,buf,nbytes); //fprintf(stderr,"ipcio_write: %d bytes\n",status); //sleep(2); } } //fclose(infd); return 0; }