static int64_t trie_store_nodes(struct trie_f *trie, struct trie_node *node) {
uint64_t i;
struct trie_node_f n = {
.prefix_off = htole64(trie->strings_off + node->prefix_off),
.children_count = node->children_count,
.values_count = htole64(node->values_count),
};
struct trie_child_entry_f *children = NULL;
int64_t node_off;
if (node->children_count) {
children = new0(struct trie_child_entry_f, node->children_count);
if (!children)
return -ENOMEM;
}
/* post-order recursion */
for (i = 0; i < node->children_count; i++) {
int64_t child_off;
child_off = trie_store_nodes(trie, node->children[i].child);
if (child_off < 0) {
free(children);
return child_off;
}
children[i].c = node->children[i].c;
children[i].child_off = htole64(child_off);
}
/* write node */
node_off = ftello(trie->f);
fwrite(&n, sizeof(struct trie_node_f), 1, trie->f);
trie->nodes_count++;
/* append children array */
if (node->children_count) {
fwrite(children, sizeof(struct trie_child_entry_f), node->children_count, trie->f);
trie->children_count += node->children_count;
free(children);
}
/* append values array */
for (i = 0; i < node->values_count; i++) {
struct trie_value_entry_f v = {
.key_off = htole64(trie->strings_off + node->values[i].key_off),
.value_off = htole64(trie->strings_off + node->values[i].value_off),
};
fwrite(&v, sizeof(struct trie_value_entry_f), 1, trie->f);
trie->values_count++;
}
return node_off;
}
static int trie_store(struct trie *trie, const char *filename) {
struct trie_f t = {
.trie = trie,
};
char *filename_tmp;
int64_t pos;
int64_t root_off;
int64_t size;
struct trie_header_f h = {
.signature = HWDB_SIG,
.tool_version = htole64(atoi(VERSION)),
.header_size = htole64(sizeof(struct trie_header_f)),
.node_size = htole64(sizeof(struct trie_node_f)),
.child_entry_size = htole64(sizeof(struct trie_child_entry_f)),
.value_entry_size = htole64(sizeof(struct trie_value_entry_f)),
};
int err;
/* calculate size of header, nodes, children entries, value entries */
t.strings_off = sizeof(struct trie_header_f);
trie_store_nodes_size(&t, trie->root);
err = fopen_temporary(filename , &t.f, &filename_tmp);
if (err < 0)
return err;
fchmod(fileno(t.f), 0444);
/* write nodes */
fseeko(t.f, sizeof(struct trie_header_f), SEEK_SET);
root_off = trie_store_nodes(&t, trie->root);
h.nodes_root_off = htole64(root_off);
pos = ftello(t.f);
h.nodes_len = htole64(pos - sizeof(struct trie_header_f));
/* write string buffer */
fwrite(trie->strings->buf, trie->strings->len, 1, t.f);
h.strings_len = htole64(trie->strings->len);
/* write header */
size = ftello(t.f);
h.file_size = htole64(size);
fseeko(t.f, 0, SEEK_SET);
fwrite(&h, sizeof(struct trie_header_f), 1, t.f);
err = ferror(t.f);
if (err)
err = -errno;
fclose(t.f);
if (err < 0 || rename(filename_tmp, filename) < 0) {
unlink(filename_tmp);
goto out;
}
log_debug("=== trie on-disk ===");
log_debug("size: %8llu bytes", (unsigned long long)size);
log_debug("header: %8zu bytes", sizeof(struct trie_header_f));
log_debug("nodes: %8llu bytes (%8llu)",
(unsigned long long)t.nodes_count * sizeof(struct trie_node_f), (unsigned long long)t.nodes_count);
log_debug("child pointers: %8llu bytes (%8llu)",
(unsigned long long)t.children_count * sizeof(struct trie_child_entry_f), (unsigned long long)t.children_count);
log_debug("value pointers: %8llu bytes (%8llu)",
(unsigned long long)t.values_count * sizeof(struct trie_value_entry_f), (unsigned long long)t.values_count);
log_debug("string store: %8llu bytes", (unsigned long long)trie->strings->len);
log_debug("strings start: %8llu", (unsigned long long) t.strings_off);
out:
free(filename_tmp);
return err;
}
static int insert_data(struct trie *trie, struct udev_list *match_list,
char *line, const char *filename) {
char *value;
struct udev_list_entry *entry;
value = strchr(line, '=');
if (!value) {
log_error("Error, key/value pair expected but got '%s' in '%s':", line, filename);
return -EINVAL;
}
value[0] = '\0';
value++;
if (line[0] == '\0' || value[0] == '\0') {
log_error("Error, empty key or value '%s' in '%s':", line, filename);
return -EINVAL;
}
udev_list_entry_foreach(entry, udev_list_get_entry(match_list))
trie_insert(trie, trie->root, udev_list_entry_get_name(entry), line, value);
return 0;
}
static int import_file(struct udev *udev, struct trie *trie, const char *filename) {
enum {
HW_MATCH,
HW_DATA,
HW_NONE,
} state = HW_NONE;
FILE *f;
char line[LINE_MAX];
struct udev_list match_list;
udev_list_init(udev, &match_list, false);
f = fopen(filename, "re");
if (f == NULL)
return -errno;
while (fgets(line, sizeof(line), f)) {
size_t len;
char *pos;
/* comment line */
if (line[0] == '#')
continue;
/* strip trailing comment */
pos = strchr(line, '#');
if (pos)
pos[0] = '\0';
/* strip trailing whitespace */
len = strlen(line);
while (len > 0 && isspace(line[len-1]))
len--;
line[len] = '\0';
switch (state) {
case HW_NONE:
if (len == 0)
break;
if (line[0] == ' ') {
log_error("Error, MATCH expected but got '%s' in '%s':", line, filename);
break;
}
/* start of record, first match */
state = HW_MATCH;
udev_list_entry_add(&match_list, line, NULL);
break;
case HW_MATCH:
if (len == 0) {
log_error("Error, DATA expected but got empty line in '%s':", filename);
state = HW_NONE;
udev_list_cleanup(&match_list);
break;
}
/* another match */
if (line[0] != ' ') {
udev_list_entry_add(&match_list, line, NULL);
break;
}
/* first data */
state = HW_DATA;
insert_data(trie, &match_list, line, filename);
break;
case HW_DATA:
/* end of record */
if (len == 0) {
state = HW_NONE;
udev_list_cleanup(&match_list);
break;
}
if (line[0] != ' ') {
log_error("Error, DATA expected but got '%s' in '%s':", line, filename);
state = HW_NONE;
udev_list_cleanup(&match_list);
break;
}
insert_data(trie, &match_list, line, filename);
break;
};
}
fclose(f);
udev_list_cleanup(&match_list);
return 0;
}
static void help(void) {
printf("Usage: udevadm hwdb OPTIONS\n"
" -u,--update update the hardware database\n"
" -t,--test=MODALIAS query database and print result\n"
" -r,--root=PATH alternative root path in the filesystem\n"
" -h,--help\n\n");
}
int DIFF_main(int argc,char *argv[])
{
int fd;
u_char *old,*_new;
off_t oldsize,newsize;
off_t *I,*V;
off_t scan,pos,len;
off_t lastscan,lastpos,lastoffset;
off_t oldscore,scsc;
off_t s,Sf,lenf,Sb,lenb;
off_t overlap,Ss,lens;
off_t i;
off_t dblen,eblen;
u_char *db,*eb;
u_char buf[8];
u_char header[32];
FILE * pf;
BZFILE * pfbz2;
int bz2err;
int bytesWritten=0;
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
that we never try to malloc(0) and get a NULL pointer */
if(((fd=open(argv[1],O_RDONLY|O_BINARY,0))<0) ||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
((old=(u_char*)malloc(oldsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,old,oldsize)!=oldsize) ||
(close(fd)==-1)) err(1,"%s",argv[1]);
if(((I=(off_t*)malloc((oldsize+1)*sizeof(off_t)))==NULL) ||
((V=(off_t*)malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL);
qsufsort(I,V,old,oldsize);
free(V);
/* Allocate newsize+1 bytes instead of newsize bytes to ensure
that we never try to malloc(0) and get a NULL pointer */
if(((fd=open(argv[2],O_RDONLY|O_BINARY,0))<0) ||
((newsize=lseek(fd,0,SEEK_END))==-1) ||
((_new=(u_char*)malloc(newsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,_new,newsize)!=newsize) ||
(close(fd)==-1)) err(1,"%s",argv[2]);
if(((db=(u_char*)malloc(newsize+1))==NULL) ||
((eb=(u_char*)malloc(newsize+1))==NULL)) err(1,NULL);
dblen=0;
eblen=0;
/* Create the patch file */
if ((pf = fopen(argv[3], "wb")) == NULL)
err(1, "%s", argv[3]);
/* Header is
0 8 "BSDIFF40"
8 8 length of bzip2ed ctrl block
16 8 length of bzip2ed diff block
24 8 length of new file */
/* File is
0 32 Header
32 ?? Bzip2ed ctrl block
?? ?? Bzip2ed diff block
?? ?? Bzip2ed extra block */
memcpy(header,"BSDIFF40",8);
offtout(0, header + 8);
offtout(0, header + 16);
offtout(newsize, header + 24);
if (fwrite(header, 32, 1, pf) != 1)
err(1, "fwrite(%s)", argv[3]);
/* Compute the differences, writing ctrl as we go */
if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL)
errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err);
scan=0;len=0;
lastscan=0;lastpos=0;lastoffset=0;
while(scan<newsize) {
oldscore=0;
for(scsc=scan+=len;scan<newsize;scan++) {
len=search(I,old,oldsize,_new+scan,newsize-scan,
0,oldsize,&pos);
for(;scsc<scan+len;scsc++)
if((scsc+lastoffset<oldsize) &&
(old[scsc+lastoffset] == _new[scsc]))
oldscore++;
if(((len==oldscore) && (len!=0)) ||
(len>oldscore+8)) break;
if((scan+lastoffset<oldsize) &&
(old[scan+lastoffset] == _new[scan]))
oldscore--;
};
if((len!=oldscore) || (scan==newsize)) {
s=0;Sf=0;lenf=0;
for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
if(old[lastpos+i]==_new[lastscan+i]) s++;
i++;
if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
};
lenb=0;
if(scan<newsize) {
s=0;Sb=0;
for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
if(old[pos-i]==_new[scan-i]) s++;
if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
};
};
if(lastscan+lenf>scan-lenb) {
overlap=(lastscan+lenf)-(scan-lenb);
s=0;Ss=0;lens=0;
for(i=0;i<overlap;i++) {
if(_new[lastscan+lenf-overlap+i]==
old[lastpos+lenf-overlap+i]) s++;
if(_new[scan-lenb+i]==
old[pos-lenb+i]) s--;
if(s>Ss) { Ss=s; lens=i+1; };
};
lenf+=lens-overlap;
lenb-=lens;
};
for(i=0;i<lenf;i++)
db[dblen+i]=_new[lastscan+i]-old[lastpos+i];
for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
eb[eblen+i]=_new[lastscan+lenf+i];
dblen+=lenf;
eblen+=(scan-lenb)-(lastscan+lenf);
offtout(lenf,buf);
BZ2_bzWrite(&bz2err, pfbz2, buf, 8);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
bytesWritten+=8;
// printf("bz2err 8 %i\n", bytesWritten);
offtout((scan-lenb)-(lastscan+lenf),buf);
BZ2_bzWrite(&bz2err, pfbz2, buf, 8);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
bytesWritten+=8;
// printf("bz2err 8 %i\n", bytesWritten);
offtout((pos-lenb)-(lastpos+lenf),buf);
BZ2_bzWrite(&bz2err, pfbz2, buf, 8);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
bytesWritten+=8;
// printf("bz2err 8 %i\n", bytesWritten);
lastscan=scan-lenb;
lastpos=pos-lenb;
lastoffset=pos-scan;
};
};
BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err);
/* Compute size of compressed ctrl data */
if ((len = ftello(pf)) == -1)
err(1, "ftello");
offtout(len-32, header + 8);
/* Write compressed diff data */
if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL)
errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err);
BZ2_bzWrite(&bz2err, pfbz2, db, dblen);
bytesWritten+=dblen;
// printf("bz2err dblen %i %i\n", dblen, bytesWritten);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err);
/* Compute size of compressed diff data */
if ((newsize = ftello(pf)) == -1)
err(1, "ftello");
offtout(newsize - len, header + 16);
/* Write compressed extra data */
if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL)
errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err);
BZ2_bzWrite(&bz2err, pfbz2, eb, eblen);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
bytesWritten+=eblen;
//printf("bz2err eblen %i %i\n", eblen, bytesWritten);
BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL);
if (bz2err != BZ_OK)
errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err);
// REMOVEME
// if ((newsize = ftello(pf)) == -1)
// err(1, "ftello");
/* Seek to the beginning, write the header, and close the file */
if (fseeko(pf, 0, SEEK_SET))
err(1, "fseeko");
if (fwrite(header, 32, 1, pf) != 1)
err(1, "fwrite(%s)", argv[3]);
if (fclose(pf))
err(1, "fclose");
/* Free the memory we used */
free(db);
free(eb);
free(I);
free(old);
free(_new);
return 0;
}
static off_t gsm_tell(struct ast_filestream *fs)
{
off_t offset = ftello(fs->f);
return (offset/GSM_FRAME_SIZE)*GSM_SAMPLES;
}
int mcfioC_OpenWriteDirect(char *filename, char *title, char *comment,
int numevts_pred, int *blkIds, u_int nBlocks)
/*
** Routine to open and write the header file for a Direct access Stream.
*/
{
int i, jstr;
off_t p1;
FILE *ff;
mcfStream *str;
if (McfStreamPtrList == NULL) {
fprintf(stderr,
" mcfio_OpenWriteDirect: We will first initialize by calling mcfio_Init.\n");
mcfioC_Init();
}
if (McfNumOfStreamActive >= MCF_STREAM_NUM_MAX) {
fprintf(stderr,
" mcfio_OpenWriteDirect: Too many streams opened simultaneously.\n");
return -1;
}
jstr = -1; i=0;
while ((jstr == -1) && (i<MCF_STREAM_NUM_MAX)) {
if (McfStreamPtrList[i] == NULL) jstr=i;
i++;
}
if(jstr == -1) {
fprintf(stderr,
" mcfio_OpenWriteDirect: Internal error, please report \n");
return -1;
}
if ((filename == NULL) || (strlen(filename) > 255)) {
fprintf(stderr,
" mcfio_OpenWriteDirect: You must give a valid UNIX filename.\n");
return -1;
}
if ((title != NULL) && (strlen(title) > 255)) {
fprintf(stderr,
" mcfio_OpenWriteDirect: Title is too long\n");
return -1;
}
if ((comment != NULL) && (strlen(comment) > 255)) {
fprintf(stderr,
" mcfio_OpenWriteDirect: comment is too long\n");
return -1;
}
/*
** Now we can try to open this file....
*/
ff = fopen(filename, "w");
if (ff == NULL) {
fprintf(stderr,
" mcfio_OpenWriteDirect: Problem opening file %s, message \n", filename);
perror ("mcfio_OpenWriteDirect");
return -1;
}
McfStreamPtrList[jstr] = (mcfStream *) malloc(sizeof(mcfStream));
str = McfStreamPtrList[jstr];
str->xdr = (XDR *) malloc(sizeof(XDR));
str->id = jstr+1;
str->row = MCFIO_WRITE;
str->dos = MCFIO_DIRECT;
str->numWordsC = 0;
str->numWordsT = 0;
str->filename = (char *) malloc(sizeof(char) * ( strlen(filename) +1) );
strcpy(str->filename,filename);
str->filePtr = ff;
str->device = NULL;
str->vsn = NULL;
str->filenumber = -1;
str->minlrec = -1;
str->maxlrec = -1;
str->shead = NULL;
str->ehead = NULL;
str->table = NULL;
str->buffer = NULL;
str->buffer2 = NULL;
xdrstdio_create(str->xdr, ff, XDR_ENCODE);
p1 = ftello(str->filePtr);
str->firstPos = p1;
str->currentPos = p1;
str->status = MCFIO_BOF;
str->fhead = (mcfxdrFileHeader *) malloc(sizeof(mcfxdrFileHeader));
/*
** Fill the file header, additional info will be written on tape
*/
if (title == NULL) strcpy(str->fhead->title,"No Title given");
else strcpy(str->fhead->title,title);
if (comment == NULL) strcpy(str->fhead->comment,"No comment");
else strcpy(str->fhead->comment, comment);
str->fhead->numevts_expect = numevts_pred;
str->fhead->numevts = 0;
/*
** Futur expansion : make this a tunable parameter.
*/
str->fhead->dimTable = MCF_DEFAULT_TABLE_SIZE;
str->fhead->firstTable = -1;
str->fhead->nBlocks = nBlocks;
if (nBlocks > 0) {
str->fhead->blockIds = (int *) malloc(sizeof(int) * nBlocks);
str->fhead->blockNames = (char**) malloc(sizeof(char *) * nBlocks);
} else {
str->fhead->blockIds = NULL;
str->fhead->blockNames = NULL;
}
for (i=0; i<nBlocks; i++) {
str->fhead->blockIds[i] = blkIds[i];
str->fhead->blockNames[i] =
(char *) malloc(sizeof(char) * (MCF_XDR_B_TITLE_LENGTH + 1));
mcfioC_GetBlockName(blkIds[i], str->fhead->blockNames[i]);
}
str->fhead->nNTuples = 0; /* Will be filled later */
if (mcfioC_Wrtfhead(str, INITIATE) == FALSE){
mcfioC_FreeStream(&McfStreamPtrList[jstr]);
fclose(ff);
return -1;
}
str->table = (mcfxdrEventTable *) malloc(sizeof(mcfxdrEventTable));
str->table->numevts=-1;
str->table->nextLocator = -1;
str->table->evtnums = (int *) malloc(sizeof(int) * str->fhead->dimTable);
str->table->storenums = (int *) malloc(sizeof(int) * str->fhead->dimTable);
str->table->runnums = (int *) malloc(sizeof(int) * str->fhead->dimTable);
str->table->trigMasks = (int *) malloc(sizeof(int) * str->fhead->dimTable);
str->table->ptrEvents =
(off_t *) calloc(str->fhead->dimTable, sizeof(off_t));
/*
** Write the first dummy table
*/
if (mcfioC_Wrttable(str, INITIATE) == FALSE) return -1;
str->ehead = (mcfxdrEventHeader *) malloc(sizeof(mcfxdrEventHeader));
str->ehead->dimBlocks = str->fhead->nBlocks;
str->ehead->nBlocks = 0;
str->ehead->dimNTuples = 0;
str->ehead->nNTuples = 0;
str->ehead->evtnum = 0;
str->ehead->previousevtnum = 0;
str->ehead->storenum = 0;
str->ehead->runnum = 0;
str->ehead->trigMask = 0;
str->ehead->nTupleIds = NULL;
str->ehead->ptrNTuples = NULL;
if (nBlocks > 0) {
str->ehead->blockIds =
(int *) malloc(sizeof(int) * str->fhead->nBlocks);
str->ehead->ptrBlocks =
(off_t *) calloc(str->fhead->nBlocks, sizeof(off_t));
} else {
str->ehead->blockIds = NULL;
str->ehead->ptrBlocks = NULL;
}
/*
** Write the first dummy event header
*/
if (mcfioC_WrtEvt(str, INITIATE) == FALSE) return -1;
str->ehead->evtnum = 0;
str->status = MCFIO_RUNNING;
McfNumOfStreamActive++;
return (jstr+1);
}
static int openReadDirect(char *filename, int mode)
/*
** Routine to open and read the header file for a Direct access Stream.
*/
{
int i, j, jstr, idtmp, ntot, ll1, jdRef, oldNumOfNTuples;
int iff;
off_t p1, p2;
FILE *ff;
mcfStream *str;
nTuDDL *ddl, *ddlRef;
struct stat statbuf;
char *srcFile;
if (McfStreamPtrList == NULL) mcfioC_Init();
if (McfNumOfStreamActive >= MCF_STREAM_NUM_MAX) {
fprintf(stderr,
" mcfio_OpenReadDirect: Too many streams opened simultaneously.\n");
return -1;
}
jstr = -1; i=0;
while ((jstr == -1) && (i<MCF_STREAM_NUM_MAX)) {
if (McfStreamPtrList[i] == NULL) jstr=i;
i++;
}
if(jstr == -1) {
fprintf(stderr,
" mcfio_OpenReadDirect: Internal error, please report \n");
return -1;
}
if ((filename == NULL) || (strlen(filename) > 255)) {
fprintf(stderr,
" mcfio_OpenReadDirect: You must give a valid UNIX filename.\n");
return -1;
}
/*
** Now we can try to open this file....
*/
if (mode == MCFIO_DIRECT) {
ff = fopen(filename, "r");
if (ff == NULL) {
fprintf(stderr,
" mcfio_OpenReadDirect: Problem opening file %s, message \n", filename);
perror ("mcfio_OpenReadDirect");
return -1;
}
} else {
/*
** Using memory mapped i/o
*/
iff = open(filename, O_RDONLY);
if (iff < 0) {
fprintf(stderr,
" mcfio_OpenReadMapped: Problem opening file %s, message \n", filename);
perror ("mcfio_OpenReadMapped");
return -1;
}
}
McfStreamPtrList[jstr] = (mcfStream *) malloc(sizeof(mcfStream));
str = McfStreamPtrList[jstr];
str->xdr = (XDR *) malloc(sizeof(XDR));
str->id = jstr+1;
str->row = MCFIO_READ;
str->dos = mode;
str->numWordsC = 0;
str->numWordsT = 0;
ll1 = strlen(filename) + 1;
str->filename = (char *) malloc(sizeof(char) * ll1);
strcpy(str->filename,filename);
if (mode == MCFIO_DIRECT) {
str->filePtr = ff;
xdrstdio_create(str->xdr, ff, XDR_DECODE);
str->fileDescr = 0;
str->fileAddr = NULL;
str->fileLen = 0;
} else {
/*
** Use memory mapped I/O
*/
if (fstat(iff, &statbuf) < 0) {
fprintf (stderr,
" mcfio_OpenReadMapped: Problem getting file length for %s \n", filename);
perror ("mcfio_OpenReadMapped");
return -1;
}
if ((srcFile =
mmap(0, statbuf.st_size, PROT_READ, MAP_FILE | MAP_SHARED, iff, 0 ))
== (caddr_t) -1) {
fprintf (stderr,
" mcfio_OpenReadMapped: Problem with memory mapping for %s \n", filename);
perror ("mcfio_OpenReadMapped");
return -1;
}
str->filePtr = (FILE *) NULL;
str->fileDescr = iff;
str->fileAddr = srcFile;
str->fileLen = (size_t) statbuf.st_size;
xdrmem_create(str->xdr, srcFile, statbuf.st_size, XDR_DECODE);
}
str->device = NULL;
str->vsn = NULL;
str->filenumber = -1;
str->minlrec = -1;
str->maxlrec = -1;
str->shead = NULL;
str->ehead = NULL;
str->table = NULL;
str->buffer = NULL;
str->buffer2 = NULL;
p1 = ftello(str->filePtr);
str->firstPos = p1;
str->status = MCFIO_BOF;
str->fhead = NULL;
oldNumOfNTuples = NumOfNTuples;
if (xdr_mcfast_fileheader(str->xdr, &idtmp,
&ntot, McfGenericVersion, &(str->fhead), str->id) == FALSE) {
fprintf (stderr,
"mcfio_OpenReadDirect: Unable to decode fileheader \n");
mcfioC_FreeStream(&McfStreamPtrList[jstr]);
mcfioC_Close(jstr+1);
return -1;
}
if (idtmp != FILEHEADER) {
fprintf (stderr,
"mcfio_OpenReadDirect: First Structure not the header \n");
fprintf (stderr,
" : Further accesses probably suspicious \n");
mcfioC_FreeStream(&McfStreamPtrList[jstr]);
mcfioC_Close(jstr+1);
return -1;
}
p2 = ftello(str->filePtr);
str->numWordsC += (ntot/4);
/*
** Check if new these Ntuple template are not reference, if so,
** set the reference pointer accordingly, conversely, recompute the
** offsets and length if requested. We also fill the sequential
** id number for the descriptors. Note: those are trivial for
** input streams, but we still fill them for consitency.
*/
for (i=0; i<str->fhead->nNTuples; i++) {
ddl = mcf_GetNTuByPtrID((oldNumOfNTuples+i+1));
if (ddl == NULL) continue;
ddl->streamId = (jstr+1);
ddl->seqNTuId = (i+1);
if (ddl->descrNtu == NULL) {
for (j=0, jdRef=1; j<i; j++, jdRef++) {
if (jdRef == ddl->referenceId) {
ddlRef = mcf_GetNTuByPtrID((oldNumOfNTuples+j+1));
/*
** back up in the linked list if need be, until we
** a fully documented descriptor.
*/
while (ddlRef->descrNtu == NULL) ddlRef = ddlRef->reference;
ddl->reference = ddlRef;
break;
}
}
} else {
if (McfNTuPleSaveDecoding == TRUE) {
mcf_ComputeNTuOffsets(ddl);
mcf_ComputeNTuLengths(ddl);
}
}
}
str->currentPos = p2;
str->fhead->firstTable = p2;
/* presumably correct , assume standard direct acces file config. */
str->numWordsT += ((p2-p1)/4);
str->status = MCFIO_RUNNING;
str->table = (mcfxdrEventTable *) malloc(sizeof(mcfxdrEventTable));
str->table->nextLocator = -1;
str->table->dim = str->fhead->dimTable;
str->table->numevts = 0;
str->table->previousnumevts = 0;
str->table->evtnums = NULL;
str->table->storenums = NULL;
str->table->runnums = NULL;
str->table->trigMasks = NULL;
str->table->ptrEvents = NULL;
str->ehead = (mcfxdrEventHeader *) malloc(sizeof(mcfxdrEventHeader));
str->ehead->dimBlocks = str->fhead->nBlocks;
str->ehead->blockIds = NULL;
str->ehead->ptrBlocks = NULL;
str->ehead->dimNTuples = str->fhead->nNTuples;
str->ehead->nTupleIds = NULL;
str->ehead->ptrNTuples = NULL;
McfNumOfStreamActive++;
return (jstr+1);
}
static int buildModel(int nolink, int nolap, int elim, int max_comp, int max_expn)
{ int first, last;
FILE *input;
DAZZ_READ *reads;
LA *local;
Pile *pile;
void *tbuffer;
int *panels, *plists;
Overlap ovl;
int64 novl;
int a, pos;
int omax, n, m;
int64 smax, tlen;
int btip, etip;
int npan, nspl, nlas;
reads = MODEL.db1->reads;
input = MODEL.input;
first = MODEL.first;
last = MODEL.last;
rewind(input);
fread(&novl,sizeof(int64),1,input);
fread(&TRACE_SPACING,sizeof(int),1,input);
if (TRACE_SPACING <= TRACE_XOVR && TRACE_SPACING != 0)
TBYTES = sizeof(uint8);
else
TBYTES = sizeof(uint16);
if (TRACE_SPACING > 0)
PANEL_SIZE = ((PANEL_TARGET-1)/TRACE_SPACING+1)*TRACE_SPACING;
else
PANEL_SIZE = ((PANEL_TARGET-1)/100+1)*100;
Read_Overlap(input,&ovl);
if (ovl.aread >= first)
MODEL.first = first = ovl.aread;
else
while (ovl.aread < first)
{ fseek(input,TBYTES*ovl.path.tlen,SEEK_CUR);
Read_Overlap(input,&ovl);
novl -= 1;
}
local = (LA *) Malloc(novl*sizeof(LA),"Allocating alignments");
if (local == NULL)
return (-1);
pile = (Pile *) Malloc(((last-first)+1)*sizeof(Pile),"Allocating piles");
if (pile == NULL)
{ free(local);
return (-1);
}
pile -= first;
npan = nspl = nlas = 0;
pos = PILE_SPACING/2;
a = first-1;
omax = m = n = 0;
smax = 0;
while (ovl.aread < last)
{ while (a < ovl.aread)
{ int j, b, e, p;
if (a >= first)
{ if (reads[a].rlen < PANEL_FUDGE)
p = 0;
else
p = ((reads[a].rlen-PANEL_FUDGE) / PANEL_SIZE);
for (j = m; j < n; j++)
{ b = local[j].abpos / PANEL_SIZE;
e = (local[j].aepos-1) / PANEL_SIZE;
if (e > p)
e = p;
nlas += e-b;
}
nspl += 1;
npan += p+1;
}
a += 1;
pile[a].first = n;
pile[a].where = pos;
pile[a].offset = ftello(input) - OvlIOSize;
pile[a].panels = npan;
if (n-m > omax)
omax = n-m;
m = n;
pos += reads[a].rlen + PILE_SPACING;
}
local[n].bread = (ovl.bread << 1);
if (COMP(ovl.flags))
local[n].bread |= 0x1;
local[n].bbpos = ovl.path.bbpos;
local[n].bepos = ovl.path.bepos;
local[n].btip = btip = tiplen(ovl.path.abpos,ovl.path.bbpos);
local[n].etip = etip = tiplen(reads[ovl.aread].rlen-ovl.path.aepos,
MODEL.db2->reads[ovl.bread].rlen-ovl.path.bepos);
local[n].abpos = ovl.path.abpos - btip;
local[n].aepos = ovl.path.aepos + etip;
local[n].toff = ftell(input);
if (CHAIN_START(ovl.flags))
local[n].etip |= STRT_FLAG;
if (CHAIN_NEXT(ovl.flags))
local[n].etip |= SUCC_FLAG;
if (ELIM(ovl.flags))
local[n].etip |= ELIM_BIT;
n += 1;
tlen = TBYTES*ovl.path.tlen;
if (smax < tlen)
smax = tlen;
fseeko(input,tlen,SEEK_CUR);
if (feof(input))
break;
if (Read_Overlap(input,&ovl))
break;
}
{ int j, b, e, p;
if (a < last-1)
{ MODEL.last = last = a+1;
pile = (Pile *) Realloc(pile+first,((last-first)+1)*sizeof(Pile),"Reallocating piles");
pile -= first;
}
if (a >= first)
{ if (reads[a].rlen < PANEL_FUDGE)
p = 0;
else
p = ((reads[a].rlen-PANEL_FUDGE) / PANEL_SIZE);
for (j = m; j < n; j++)
{ b = local[j].abpos / PANEL_SIZE;
e = (local[j].aepos-1) / PANEL_SIZE;
if (e > p)
e = p;
nlas += e-b;
}
nspl += 1;
npan += p+1;
}
pile[last].first = n;
pile[last].where = pos;
pile[last].offset = ftello(input) - OvlIOSize;
pile[last].panels = npan;
if (n-m > omax)
omax = n-m;
pile[last].where -= PILE_SPACING/2;
pile[last].offset += OvlIOSize;
if (n < novl)
local = (LA *) Realloc(local,n*sizeof(LA),"Allocating alignments");
}
{ LA *temp;
nlas = n + nlas;
panels = (int *) Malloc((npan+1)*sizeof(int),"Panels");
if (panels == NULL)
goto error;
plists = (int *) Malloc(nlas*sizeof(int),"Panel Lists");
if (plists == NULL)
goto error;
temp = (LA *) Realloc(local,(n+1)*sizeof(LA),"Finalizing alignments");
if (temp == NULL)
goto error;
local = temp;
tbuffer = Malloc(smax,"Allocating trace buffer");
if (tbuffer == NULL)
goto error;
}
{ int a, k;
int *manels;
int *count;
for (k = 0; k <= npan; k++)
panels[k] = 0;
manels = panels+1;
for (a = first; a < last; a++)
{ int j, b, e, p;
count = manels + pile[a].panels;
if (reads[a].rlen < PANEL_FUDGE)
p = 0;
else
p = ((reads[a].rlen-PANEL_FUDGE) / PANEL_SIZE);
for (j = pile[a].first; j < pile[a+1].first; j++)
{ b = local[j].abpos / PANEL_SIZE;
e = (local[j].aepos-1) / PANEL_SIZE;
if (e > p)
e = p;
for (k = b; k <= e; k++)
count[k] += 1;
}
}
for (k = 2; k <= npan; k++)
panels[k] += panels[k-1];
for (a = first; a < last; a++)
{ int j, b, e, p;
count = panels + pile[a].panels;
if (reads[a].rlen < PANEL_FUDGE)
p = 0;
else
p = ((reads[a].rlen-PANEL_FUDGE) / PANEL_SIZE);
p = ((reads[a].rlen-PANEL_FUDGE) / PANEL_SIZE);
for (j = pile[a].first; j < pile[a+1].first; j++)
{ b = local[j].abpos / PANEL_SIZE;
e = (local[j].aepos-1) / PANEL_SIZE;
if (e > p)
e = p;
for (k = b; k <= e; k++)
plists[count[k]++] = j;
local[j].abpos += local[j].btip;
local[j].aepos -= (local[j].etip & DATA_ETIP);
}
}
for (k = npan; k >= 1; k--)
panels[k] = panels[k-1];
panels[0] = 0;
}
MODEL.local = local;
MODEL.pile = pile;
MODEL.omax = omax;
MODEL.tbuf = tbuffer;
MODEL.tspace = TRACE_SPACING;
MODEL.tbytes = TBYTES;
MODEL.panels = panels;
MODEL.plists = plists;
UNDEFINED = 0;
if (reLayoutModel(nolink,nolap,elim,max_comp,max_expn))
{ UNDEFINED = 1;
goto error;
}
return (0);
error:
free(local);
free(pile+first);
return (1);
}
static int
add_data(struct zip *za, struct zip_source *zs, struct zip_dirent *de, FILE *ft)
{
off_t offstart, offend;
zip_source_callback cb;
void *ud;
struct zip_stat st;
cb = zs->f;
ud = zs->ud;
if (cb(ud, &st, sizeof(st), ZIP_SOURCE_STAT) < (ssize_t)sizeof(st)) {
ch_set_error(&za->error, cb, ud);
return -1;
}
if (cb(ud, NULL, 0, ZIP_SOURCE_OPEN) < 0) {
ch_set_error(&za->error, cb, ud);
return -1;
}
offstart = ftello(ft);
if (_zip_dirent_write(de, ft, 1, &za->error) < 0)
return -1;
if (st.comp_method != ZIP_CM_STORE) {
if (add_data_comp(cb, ud, &st, ft, &za->error) < 0)
return -1;
}
else {
if (add_data_uncomp(za, cb, ud, &st, ft) < 0)
return -1;
}
if (cb(ud, NULL, 0, ZIP_SOURCE_CLOSE) < 0) {
ch_set_error(&za->error, cb, ud);
return -1;
}
offend = ftello(ft);
if (fseeko(ft, offstart, SEEK_SET) < 0) {
_zip_error_set(&za->error, ZIP_ER_SEEK, errno);
return -1;
}
de->last_mod = st.mtime;
de->comp_method = st.comp_method;
de->crc = st.crc;
de->uncomp_size = st.size;
de->comp_size = st.comp_size;
if (zip_get_archive_flag(za, ZIP_AFL_TORRENT, 0))
_zip_dirent_torrent_normalize(de);
if (_zip_dirent_write(de, ft, 1, &za->error) < 0)
return -1;
if (fseeko(ft, offend, SEEK_SET) < 0) {
_zip_error_set(&za->error, ZIP_ER_SEEK, errno);
return -1;
}
return 0;
}
static off_t h263_tell(struct ast_filestream *fs)
{
off_t offset = ftello(fs->f);
return offset; /* XXX totally bogus, needs fixing */
}
/*
Open one file, probe to see if there is several file with contiguous name
and handle them as one big file if that's the case
If multi is set to probe, return value will be APPEND if there is several files, dont_append if one
if multi is set to dont_append, file won't be auto appended even if they exist
*/
uint8_t fileParser::open( const char *filename,FP_TYPE *multi )
{
uint32_t decimals = 0; // number of decimals
char *left = NULL, *right = NULL; // parts of filename (after splitting)
uint8_t count = 0; // number of follow-ups
uint32_t base=0;
bool splitFile=false;
// int i = 0;
// index (general use)
if(*multi!=FP_DONT_APPEND)
{
aprintf("Checking if there are several files...\n");
splitFile=ADM_splitSequencedFile(filename, &left, &right,&decimals,&base);
if(splitFile)
{
aprintf("left:<%s>, right=<%s>,base=%" PRIu32",digit=%" PRIu32"\n",left,right,base,decimals);
}else
{
aprintf("No.\n");
}
}
// ____________________
// Single loading
// ____________________
if( false ==splitFile )
{
fdIo newFd;
aprintf( "\nSimple loading: \n" );
_curFd = 0;
FILE *f=NULL;
// open file
if(! (f = ADM_fopen(filename, "rb")) )
{ return 0; }
newFd.file=f;
// calculate file-size
fseeko( f, 0, SEEK_END );
newFd.fileSize = ftello( f );
fseeko( f, 0, SEEK_SET );
newFd.fileSizeCumul=0;
_size=newFd.fileSize;
listOfFd.append(newFd);
aprintf( " file: %s, size: %" PRIu64"\n", filename, newFd.fileSize );
aprintf( " found 1 files \n" );
aprintf( "Done \n" );
return 1;
}
// ____________________
// Multi loading
// ____________________
uint32_t tabSize;
std::string leftPart(left);
std::string rightPart(right);
delete [] left;
delete [] right;
left=NULL;
right=NULL;
aprintf( "\nAuto adding: \n" );
uint32_t current=base;
_curFd = 0;
uint64_t total=0;
// build match string
char match[16];
match[0]='%';
match[1]='0';
sprintf(match+2,"%d",decimals); // snprintf instead ...
strcat(match,"d");
match[15]=0;
aprintf("Using %s as match string\n",match);
char number[16];
while(1)
{
sprintf(number,match,current);
std::string middle(number);
std::string outName=leftPart+middle+rightPart;
aprintf("Checking %s\n",outName.c_str());
// open file
FILE *f= ADM_fopen(outName.c_str(), "rb");
if(!f)
{
// we need at least one file!
if( !count )
{ return 0; }
else
{
printf( " file: %s not found. \n", outName.c_str() );
break;
}
}
// calculate file-size
fdIo myFd;
myFd.file=f;
myFd.fileSize=ADM_fileSize(outName.c_str());
myFd.fileSizeCumul = total;
total+= myFd.fileSize;
aprintf( " file %d: %s, size: %" PRIu64"\n", (count + 1), outName.c_str(),
myFd.fileSize );
listOfFd.append(myFd);
count++;
current++;
}
_size=total;
// clean up
if(*multi==FP_PROBE)
{
if(count>1)
*multi=FP_APPEND; //
else
*multi=FP_DONT_APPEND;
}
aprintf( " found %d files \n", count );
aprintf( "Done \n" );
return 1;
} // fileParser::open()
/* Function: p7_oprofile_ReadMSVInfo()
* Synopsis: Read MSV filter info, but not the scores.
* Incept: MSF, Thu Oct 15, 2009 [Janelia]
*
* Purpose: Read just enough of the MSV filter header from the
* <.h3f> file associated with an open HMM file <hfp>
* to skip ahead to the next MSV filter. Allocate a new
* model, populate it with just the file offsets of this
* model and return a pointer to it in <*ret_om>.
*
* The <.h3f> file was opened automatically, if it existed,
* when the HMM file was opened with <p7_hmmfile_Open()>.
*
* When no more HMMs remain in the file, return <eslEOF>.
*
* Args: hfp - open HMM file, with associated .h3p file
* byp_abc - BYPASS: <*byp_abc == ESL_ALPHABET *> if known;
* <*byp_abc == NULL> if desired;
* <NULL> if unwanted.
* ret_om - RETURN: newly allocated <om> with partial MSV
* filter data filled in.
*
* Returns: <eslOK> on success. <*ret_om> is allocated here;
* caller free's with <p7_oprofile_Destroy()>.
* <*byp_abc> is allocated here if it was requested;
* caller free's with <esl_alphabet_Destroy()>.
*
* Returns <eslEFORMAT> if <hfp> has no <.h3f> file open,
* or on any parsing error.
*
* Returns <eslEINCOMPAT> if the HMM we read is incompatible
* with the existing alphabet <*byp_abc> led us to expect.
*
* On any returned error, <hfp->errbuf> contains an
* informative error message.
*
* Throws: <eslEMEM> on allocation error.
*/
int
p7_oprofile_ReadInfoMSV(P7_HMMFILE *hfp, ESL_ALPHABET **byp_abc, P7_OPROFILE **ret_om)
{
P7_OPROFILE *om = NULL;
ESL_ALPHABET *abc = NULL;
uint32_t magic;
off_t roff;
int M, Q16;
int n;
int alphatype;
int status;
if (hfp->errbuf != NULL) hfp->errbuf[0] = '\0';
if (hfp->ffp == NULL) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "no MSV profile file; hmmpress probably wasn't run");
if (feof(hfp->ffp)) { status = eslEOF; goto ERROR; } /* normal EOF: no more profiles */
/* keep track of the starting offset of the MSV model */
roff = ftello(hfp->ffp);
if (! fread( (char *) &magic, sizeof(uint32_t), 1, hfp->ffp)) { status = eslEOF; goto ERROR; }
if (magic == v3a_fmagic) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "this is an outdated HMM database (3/a format); please hmmpress your HMM file again");
if (magic != v3b_fmagic) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "bad magic; not an HMM database?");
if (! fread( (char *) &M, sizeof(int), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read model size M");
if (! fread( (char *) &alphatype, sizeof(int), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read alphabet type");
Q16 = p7O_NQB(M);
/* Set or verify alphabet. */
if (byp_abc == NULL || *byp_abc == NULL) { /* alphabet unknown: whether wanted or unwanted, make a new one */
if ((abc = esl_alphabet_Create(alphatype)) == NULL) ESL_XFAIL(eslEMEM, hfp->errbuf, "allocation failed: alphabet");
} else { /* alphabet already known: verify it against what we see in the HMM */
abc = *byp_abc;
if (abc->type != alphatype)
ESL_XFAIL(eslEINCOMPAT, hfp->errbuf, "Alphabet type mismatch: was %s, but current profile says %s",
esl_abc_DecodeType(abc->type), esl_abc_DecodeType(alphatype));
}
/* Now we know the sizes of things, so we can allocate. */
if ((om = p7_oprofile_Create(M, abc)) == NULL) ESL_XFAIL(eslEMEM, hfp->errbuf, "allocation failed: oprofile");
om->M = M;
om->roff = roff;
/* calculate the remaining length of the msv model */
om->name = NULL;
if (!fread((char *) &n, sizeof(int), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read name length");
roff += (sizeof(int) * 3); /* magic, model size, and alphabet type */
roff += sizeof(int) + n + 1; /* length, name string and terminator '\0' */
roff += (sizeof(float) + sizeof(uint8_t) * 5); /* transition costs, bias, scale and base */
roff += (sizeof(vector signed char) * abc->Kp * Q16); /* msv scores */
roff += (sizeof(float) * p7_NEVPARAM); /* stat params */
roff += (sizeof(off_t) * p7_NOFFSETS); /* hmmpfam offsets */
roff += (sizeof(float) * p7_MAXABET); /* model composition */
/* keep track of the ending offset of the MSV model */
p7_oprofile_Position(hfp, roff);
om->eoff = ftello(hfp->ffp) - 1;
if (byp_abc != NULL) *byp_abc = abc; /* pass alphabet (whether new or not) back to caller, if caller wanted it */
*ret_om = om;
return eslOK;
ERROR:
if (abc != NULL && (byp_abc == NULL || *byp_abc == NULL)) esl_alphabet_Destroy(abc); /* destroy alphabet if we created it here */
if (om != NULL) p7_oprofile_Destroy(om);
*ret_om = NULL;
return status;
}
static void
utest_ReadWrite(P7_HMM *hmm, P7_OPROFILE *om)
{
char *msg = "oprofile read/write unit test failure";
ESL_ALPHABET *abc = NULL;
P7_OPROFILE *om2 = NULL;
char tmpfile[16] = "esltmpXXXXXX";
char *mfile = NULL;
char *ffile = NULL;
char *pfile = NULL;
char *ssifile = NULL;
FILE *fp = NULL;
FILE *mfp = NULL;
FILE *ffp = NULL;
FILE *pfp = NULL;
ESL_NEWSSI *nssi = NULL;
P7_HMMFILE *hfp = NULL;
uint16_t fh = 0;
float tolerance = 0.001;
char errbuf[eslERRBUFSIZE];
/* 1. A mini version of hmmpress: save the test HMM to a file along with its associated .h3{mfpi} files
*/
if ( esl_tmpfile_named(tmpfile, &fp) != eslOK) esl_fatal(msg);
if ( esl_sprintf(&mfile, "%s.h3m", tmpfile) != eslOK) esl_fatal(msg);
if ( esl_sprintf(&ffile, "%s.h3f", tmpfile) != eslOK) esl_fatal(msg);
if ( esl_sprintf(&pfile, "%s.h3p", tmpfile) != eslOK) esl_fatal(msg);
if ( esl_sprintf(&ssifile, "%s.h3i", tmpfile) != eslOK) esl_fatal(msg);
if ( esl_newssi_Open(ssifile, TRUE, &nssi) != eslOK) esl_fatal(msg);
if (( mfp = fopen(mfile, "wb")) == NULL) esl_fatal(msg);
if (( ffp = fopen(ffile, "wb")) == NULL) esl_fatal(msg);
if (( pfp = fopen(pfile, "wb")) == NULL) esl_fatal(msg);
/* the disk offsets are all 0 by construction, if there's only one
* HMM in the file - but don't want to forget them, if we change the
* unit test in the future to be multi HMM
*/
if ((om->offs[p7_MOFFSET] = ftello(mfp)) == -1) esl_fatal(msg);
if ((om->offs[p7_FOFFSET] = ftello(ffp)) == -1) esl_fatal(msg);
if ((om->offs[p7_POFFSET] = ftello(pfp)) == -1) esl_fatal(msg);
if ( p7_hmmfile_WriteASCII(fp, -1, hmm) != eslOK) esl_fatal(msg);
if ( p7_hmmfile_WriteBinary(mfp, -1, hmm) != eslOK) esl_fatal(msg);
if ( p7_oprofile_Write(ffp, pfp, om) != eslOK) esl_fatal(msg);
if ( esl_newssi_AddFile(nssi, tmpfile, 0, &fh) != eslOK) esl_fatal(msg);
if ( esl_newssi_AddKey (nssi, hmm->name, fh, om->offs[p7_MOFFSET], 0, 0) != eslOK) esl_fatal(msg);
if ( esl_newssi_Write(nssi) != eslOK) esl_fatal(msg);
fclose(fp);
fclose(mfp);
fclose(ffp);
fclose(pfp);
esl_newssi_Close(nssi);
/* 2. read the optimized profile back in */
if ( p7_hmmfile_Open(tmpfile, NULL, &hfp) != eslOK) esl_fatal(msg);
if ( p7_oprofile_ReadMSV(hfp, &abc, &om2) != eslOK) esl_fatal(msg);
if ( p7_oprofile_ReadRest(hfp, om2) != eslOK) esl_fatal(msg);
/* 3. it should be identical to the original */
if ( p7_oprofile_Compare(om, om2, tolerance, errbuf) != eslOK) esl_fatal("%s\n%s", msg, errbuf);
p7_oprofile_Destroy(om2);
p7_hmmfile_Close(hfp);
esl_alphabet_Destroy(abc);
remove(ssifile);
remove(ffile);
remove(pfile);
remove(mfile);
remove(tmpfile);
free(ssifile);
free(mfile);
free(ffile);
free(pfile);
}
/* Function: p7_oprofile_ReadMSV()
* Synopsis: Read MSV filter part of an optimized profile.
* Incept: SRE, Wed Jan 21 10:39:20 2009 [Janelia]
*
* Purpose: Read the MSV filter part of a profile from the
* <.h3f> file associated with an open HMM file <hfp>.
* Allocate a new model, populate it with this minimal
* MSV filter information, and return a pointer to it
* in <*ret_om>.
*
* Our alphabet may get set by the first HMM we read. If
* <*byp_abc> is <NULL> at start, create a new alphabet and
* return a pointer to it in <*byp_abc>. If <*byp_abc> is
* non-<NULL>, it is assumed to be a pointer to an existing
* alphabet; we verify that the HMM's alphabet matches it
* and <*ret_abc> isn't changed. This is the same
* convention used by <p7_hmmfile_Read()>.
*
* The <.h3f> file was opened automatically, if it existed,
* when the HMM file was opened with <p7_hmmfile_Open()>.
*
* When no more HMMs remain in the file, return <eslEOF>.
*
* Args: hfp - open HMM file, with associated .h3p file
* byp_abc - BYPASS: <*byp_abc == ESL_ALPHABET *> if known;
* <*byp_abc == NULL> if desired;
* <NULL> if unwanted.
* ret_om - RETURN: newly allocated <om> with MSV filter
* data filled in.
*
* Returns: <eslOK> on success. <*ret_om> is allocated here;
* caller free's with <p7_oprofile_Destroy()>.
* <*byp_abc> is allocated here if it was requested;
* caller free's with <esl_alphabet_Destroy()>.
*
* Returns <eslEFORMAT> if <hfp> has no <.h3f> file open,
* or on any parsing error.
*
* Returns <eslEINCOMPAT> if the HMM we read is incompatible
* with the existing alphabet <*byp_abc> led us to expect.
*
* On any returned error, <hfp->errbuf> contains an
* informative error message.
*
* Throws: <eslEMEM> on allocation error.
*/
int
p7_oprofile_ReadMSV(P7_HMMFILE *hfp, ESL_ALPHABET **byp_abc, P7_OPROFILE **ret_om)
{
P7_OPROFILE *om = NULL;
ESL_ALPHABET *abc = NULL;
uint32_t magic;
off_t roff;
int M, Q16;
int x,n;
int alphatype;
int status;
if (hfp->errbuf != NULL) hfp->errbuf[0] = '\0';
if (hfp->ffp == NULL) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "no MSV profile file; hmmpress probably wasn't run");
if (feof(hfp->ffp)) { status = eslEOF; goto ERROR; } /* normal EOF: no more profiles */
/* keep track of the starting offset of the MSV model */
roff = ftello(hfp->ffp);
if (! fread( (char *) &magic, sizeof(uint32_t), 1, hfp->ffp)) { status = eslEOF; goto ERROR; }
if (magic == v3a_fmagic) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "this is an outdated HMM database (3/a format); please hmmpress your HMM file again");
if (magic != v3b_fmagic) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "bad magic; not an HMM database?");
if (! fread( (char *) &M, sizeof(int), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read model size M");
if (! fread( (char *) &alphatype, sizeof(int), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read alphabet type");
Q16 = p7O_NQB(M);
/* Set or verify alphabet. */
if (byp_abc == NULL || *byp_abc == NULL) { /* alphabet unknown: whether wanted or unwanted, make a new one */
if ((abc = esl_alphabet_Create(alphatype)) == NULL) ESL_XFAIL(eslEMEM, hfp->errbuf, "allocation failed: alphabet");
} else { /* alphabet already known: verify it against what we see in the HMM */
abc = *byp_abc;
if (abc->type != alphatype)
ESL_XFAIL(eslEINCOMPAT, hfp->errbuf, "Alphabet type mismatch: was %s, but current profile says %s",
esl_abc_DecodeType(abc->type), esl_abc_DecodeType(alphatype));
}
/* Now we know the sizes of things, so we can allocate. */
if ((om = p7_oprofile_Create(M, abc)) == NULL) ESL_XFAIL(eslEMEM, hfp->errbuf, "allocation failed: oprofile");
om->M = M;
om->roff = roff;
if (! fread((char *) &n, sizeof(int), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read name length");
ESL_ALLOC(om->name, sizeof(char) * (n+1));
if (! fread((char *) om->name, sizeof(char), n+1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read name");
if (! fread((char *) &(om->tbm_b), sizeof(uint8_t), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read tbm");
if (! fread((char *) &(om->tec_b), sizeof(uint8_t), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read tec");
if (! fread((char *) &(om->tjb_b), sizeof(uint8_t), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read tjb");
if (! fread((char *) &(om->scale_b),sizeof(float), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read scale");
if (! fread((char *) &(om->base_b), sizeof(uint8_t), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read base");
if (! fread((char *) &(om->bias_b), sizeof(uint8_t), 1, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read bias");
for (x = 0; x < abc->Kp; x++)
if (! fread((char *) om->rbv[x], sizeof(vector unsigned char), Q16, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read msv scores at %d [residue %c]", x, abc->sym[x]);
if (! fread((char *) om->evparam, sizeof(float), p7_NEVPARAM, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read stat params");
if (! fread((char *) om->offs, sizeof(off_t), p7_NOFFSETS, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read hmmpfam offsets");
if (! fread((char *) om->compo, sizeof(float), p7_MAXABET, hfp->ffp)) ESL_XFAIL(eslEFORMAT, hfp->errbuf, "failed to read model composition");
/* keep track of the ending offset of the MSV model */
om->eoff = ftello(hfp->ffp) - 1;;
if (byp_abc != NULL) *byp_abc = abc; /* pass alphabet (whether new or not) back to caller, if caller wanted it */
*ret_om = om;
return eslOK;
ERROR:
if (abc != NULL && (byp_abc == NULL || *byp_abc == NULL)) esl_alphabet_Destroy(abc); /* destroy alphabet if we created it here */
if (om != NULL) p7_oprofile_Destroy(om);
*ret_om = NULL;
return status;
}
int main(int argc, char **argv) {
aom_codec_ctx_t codec;
AvxVideoReader *reader = NULL;
AvxVideoWriter *writer = NULL;
const AvxInterface *decoder = NULL;
const AvxVideoInfo *info = NULL;
int num_references;
int i;
aom_codec_pts_t pts;
const char *tile_list_file = NULL;
exec_name = argv[0];
if (argc != 5) die("Invalid number of arguments.");
reader = aom_video_reader_open(argv[1]);
if (!reader) die("Failed to open %s for reading.", argv[1]);
num_references = (int)strtol(argv[3], NULL, 0);
info = aom_video_reader_get_info(reader);
// The writer to write out ivf file in tile list OBU, which can be decoded by
// AV1 decoder.
writer = aom_video_writer_open(argv[2], kContainerIVF, info);
if (!writer) die("Failed to open %s for writing", argv[2]);
tile_list_file = argv[4];
decoder = get_aom_decoder_by_fourcc(info->codec_fourcc);
if (!decoder) die("Unknown input codec.");
printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface()));
if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0))
die_codec(&codec, "Failed to initialize decoder.");
// Decode anchor frames.
aom_codec_control_(&codec, AV1_SET_TILE_MODE, 0);
printf("Reading %d reference images.\n", num_references);
for (i = 0; i < num_references; ++i) {
aom_video_reader_read_frame(reader);
size_t frame_size = 0;
const unsigned char *frame =
aom_video_reader_get_frame(reader, &frame_size);
pts = (aom_codec_pts_t)aom_video_reader_get_frame_pts(reader);
// Copy references bitstream directly.
if (!aom_video_writer_write_frame(writer, frame, frame_size, pts))
die_codec(&codec, "Failed to copy compressed anchor frame.");
if (aom_codec_decode(&codec, frame, frame_size, NULL))
die_codec(&codec, "Failed to decode frame.");
}
// Decode camera frames.
aom_codec_control_(&codec, AV1_SET_TILE_MODE, 1);
aom_codec_control_(&codec, AV1D_EXT_TILE_DEBUG, 1);
FILE *infile = aom_video_reader_get_file(reader);
// Record the offset of the first camera image.
const FileOffset camera_frame_pos = ftello(infile);
printf("Loading compressed frames into memory.\n");
// Count the frames in the lightfield.
int num_frames = 0;
while (aom_video_reader_read_frame(reader)) {
++num_frames;
}
if (num_frames < 1) die("Input light field has no frames.");
// Read all of the lightfield frames into memory.
unsigned char **frames =
(unsigned char **)malloc(num_frames * sizeof(unsigned char *));
size_t *frame_sizes = (size_t *)malloc(num_frames * sizeof(size_t));
// Seek to the first camera image.
fseeko(infile, camera_frame_pos, SEEK_SET);
for (int f = 0; f < num_frames; ++f) {
aom_video_reader_read_frame(reader);
size_t frame_size = 0;
const unsigned char *frame =
aom_video_reader_get_frame(reader, &frame_size);
frames[f] = (unsigned char *)malloc(frame_size * sizeof(unsigned char));
memcpy(frames[f], frame, frame_size);
frame_sizes[f] = frame_size;
}
printf("Read %d frames.\n", num_frames);
// Copy first camera frame for getting camera frame header. This is done
// only once.
{
size_t frame_size = frame_sizes[0];
const unsigned char *frame = frames[0];
pts = num_references;
aom_tile_data frame_header_info = { 0, NULL, 0 };
// Need to decode frame header to get camera frame header info. So, here
// decoding 1 tile is enough.
aom_codec_control_(&codec, AV1_SET_DECODE_TILE_ROW, 0);
aom_codec_control_(&codec, AV1_SET_DECODE_TILE_COL, 0);
aom_codec_err_t aom_status =
aom_codec_decode(&codec, frame, frame_size, NULL);
if (aom_status) die_codec(&codec, "Failed to decode tile.");
aom_codec_control_(&codec, AV1D_GET_FRAME_HEADER_INFO, &frame_header_info);
size_t obu_size_offset =
(uint8_t *)frame_header_info.coded_tile_data - frame;
size_t length_field_size = frame_header_info.coded_tile_data_size;
// Remove ext-tile tile info.
uint32_t frame_header_size = (uint32_t)frame_header_info.extra_size - 1;
size_t bytes_to_copy =
obu_size_offset + length_field_size + frame_header_size;
unsigned char *frame_hdr_buf = (unsigned char *)malloc(bytes_to_copy);
if (frame_hdr_buf == NULL)
die_codec(&codec, "Failed to allocate frame header buffer.");
memcpy(frame_hdr_buf, frame, bytes_to_copy);
// Update frame header OBU size.
size_t bytes_written = 0;
if (aom_uleb_encode_fixed_size(
frame_header_size, length_field_size, length_field_size,
frame_hdr_buf + obu_size_offset, &bytes_written))
die_codec(&codec, "Failed to encode the tile list obu size.");
// Copy camera frame header bitstream.
if (!aom_video_writer_write_frame(writer, frame_hdr_buf, bytes_to_copy,
pts))
die_codec(&codec, "Failed to copy compressed camera frame header.");
free(frame_hdr_buf);
}
// Read out the image format.
aom_img_fmt_t ref_fmt = 0;
if (aom_codec_control(&codec, AV1D_GET_IMG_FORMAT, &ref_fmt))
die_codec(&codec, "Failed to get the image format");
const int bps = get_image_bps(ref_fmt);
if (!bps) die_codec(&codec, "Invalid image format.");
// read out the tile size.
unsigned int tile_size = 0;
if (aom_codec_control(&codec, AV1D_GET_TILE_SIZE, &tile_size))
die_codec(&codec, "Failed to get the tile size");
const unsigned int tile_width = tile_size >> 16;
const unsigned int tile_height = tile_size & 65535;
// Allocate a buffer to store tile list bitstream.
const size_t data_sz = MAX_TILES * ALIGN_POWER_OF_TWO(tile_width, 5) *
ALIGN_POWER_OF_TWO(tile_height, 5) * bps / 8;
unsigned char *tl_buf = (unsigned char *)malloc(data_sz);
if (tl_buf == NULL) die_codec(&codec, "Failed to allocate tile list buffer.");
aom_codec_pts_t tl_pts = num_references;
const uint8_t output_frame_width_in_tiles_minus_1 =
output_frame_width / tile_width - 1;
const uint8_t output_frame_height_in_tiles_minus_1 =
output_frame_height / tile_height - 1;
printf("Reading tile list from file.\n");
char line[1024];
FILE *tile_list_fptr = fopen(tile_list_file, "r");
if (!tile_list_fptr) die_codec(&codec, "Failed to open tile list file.");
int num_tiles = 0;
TILE_LIST_INFO tiles[MAX_TILES];
while ((fgets(line, 1024, tile_list_fptr)) != NULL) {
if (line[0] == 'F' || num_tiles >= MAX_TILES) {
// Flush existing tile list and start another, either because we hit a
// new render frame or because we've hit our max number of tiles per list.
if (num_tiles > 0) {
process_tile_list(tiles, num_tiles, tl_pts, frames, frame_sizes, &codec,
tl_buf, writer, output_frame_width_in_tiles_minus_1,
output_frame_height_in_tiles_minus_1);
++tl_pts;
}
num_tiles = 0;
}
if (line[0] == 'F') {
continue;
}
if (sscanf(line, "%d %d %d %d", &tiles[num_tiles].image_idx,
&tiles[num_tiles].reference_idx, &tiles[num_tiles].tile_col,
&tiles[num_tiles].tile_row) == 4) {
if (tiles[num_tiles].image_idx >= num_frames) {
die("Tile list image_idx out of bounds: %d >= %d.",
tiles[num_tiles].image_idx, num_frames);
}
if (tiles[num_tiles].reference_idx >= num_references) {
die("Tile list reference_idx out of bounds: %d >= %d.",
tiles[num_tiles].reference_idx, num_references);
}
++num_tiles;
}
}
if (num_tiles > 0) {
// Flush out the last tile list.
process_tile_list(tiles, num_tiles, tl_pts, frames, frame_sizes, &codec,
tl_buf, writer, output_frame_width_in_tiles_minus_1,
output_frame_height_in_tiles_minus_1);
++tl_pts;
}
const int num_tile_lists = (int)(tl_pts - pts);
printf("Finished processing tile lists. Num tile lists: %d.\n",
num_tile_lists);
free(tl_buf);
for (int f = 0; f < num_frames; ++f) {
free(frames[f]);
}
free(frame_sizes);
free(frames);
if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec");
aom_video_writer_close(writer);
aom_video_reader_close(reader);
return EXIT_SUCCESS;
}
/**
* Read a range from file.
* @param remain: Rest of line (after RANGE keyword).
* @param in: file to read from.
* @param name: name to print in errors.
* @param pstate: read state structure with
* with lineno : incremented as lines are read.
* ttl, origin, prev for readentry.
* @param line: line buffer.
* @return: range object to add to list, or NULL on error.
*/
static struct replay_range*
replay_range_read(char* remain, FILE* in, const char* name,
struct sldns_file_parse_state* pstate, char* line)
{
struct replay_range* rng = (struct replay_range*)malloc(
sizeof(struct replay_range));
off_t pos;
char *parse;
struct entry* entry, *last = NULL;
if(!rng)
return NULL;
memset(rng, 0, sizeof(*rng));
/* read time range */
if(sscanf(remain, " %d %d", &rng->start_step, &rng->end_step)!=2) {
log_err("Could not read time range: %s", line);
free(rng);
return NULL;
}
/* read entries */
pos = ftello(in);
while(fgets(line, MAX_LINE_LEN-1, in)) {
pstate->lineno++;
parse = line;
while(isspace((int)*parse))
parse++;
if(!*parse || *parse == ';') {
pos = ftello(in);
continue;
}
if(parse_keyword(&parse, "ADDRESS")) {
while(isspace((int)*parse))
parse++;
strip_end_white(parse);
if(!extstrtoaddr(parse, &rng->addr, &rng->addrlen)) {
log_err("Line %d: could not read ADDRESS: %s",
pstate->lineno, parse);
free(rng);
return NULL;
}
pos = ftello(in);
continue;
}
if(parse_keyword(&parse, "RANGE_END")) {
return rng;
}
/* set position before line; read entry */
pstate->lineno--;
fseeko(in, pos, SEEK_SET);
entry = read_entry(in, name, pstate, 1);
if(!entry)
fatal_exit("%d: bad entry", pstate->lineno);
entry->next = NULL;
if(last)
last->next = entry;
else rng->match = entry;
last = entry;
pos = ftello(in);
}
replay_range_delete(rng);
return NULL;
}
/**
* Imports a base64 encoded signature into a MAR file
*
* @param src The path of the source MAR file
* @param sigIndex The index of the signature to import
* @param base64SigFile A file which contains the signature to import
* @param dest The path of the destination MAR file with replaced signature
* @return 0 on success
* -1 on error
*/
int
import_signature(const char *src, uint32_t sigIndex,
const char *base64SigFile, const char *dest)
{
int rv = -1;
FILE *fpSrc = NULL;
FILE *fpDest = NULL;
FILE *fpSigFile = NULL;
uint32_t i;
uint32_t signatureCount, signatureLen, signatureAlgorithmID,
numChunks, leftOver;
char buf[BLOCKSIZE];
uint64_t sizeOfSrcMAR, sizeOfBase64EncodedFile;
char *passedInSignatureB64 = NULL;
uint8_t *passedInSignatureRaw = NULL;
uint8_t *extractedMARSignature = NULL;
unsigned int passedInSignatureLenRaw;
if (!src || !dest) {
fprintf(stderr, "ERROR: Invalid parameter passed in.\n");
goto failure;
}
fpSrc = fopen(src, "rb");
if (!fpSrc) {
fprintf(stderr, "ERROR: could not open source file: %s\n", src);
goto failure;
}
fpDest = fopen(dest, "wb");
if (!fpDest) {
fprintf(stderr, "ERROR: could not open dest file: %s\n", dest);
goto failure;
}
fpSigFile = fopen(base64SigFile , "rb");
if (!fpSigFile) {
fprintf(stderr, "ERROR: could not open sig file: %s\n", base64SigFile);
goto failure;
}
/* Get the src file size */
if (fseeko(fpSrc, 0, SEEK_END)) {
fprintf(stderr, "ERROR: Could not seek to end of src file.\n");
goto failure;
}
sizeOfSrcMAR = ftello(fpSrc);
if (fseeko(fpSrc, 0, SEEK_SET)) {
fprintf(stderr, "ERROR: Could not seek to start of src file.\n");
goto failure;
}
/* Get the sig file size */
if (fseeko(fpSigFile, 0, SEEK_END)) {
fprintf(stderr, "ERROR: Could not seek to end of sig file.\n");
goto failure;
}
sizeOfBase64EncodedFile= ftello(fpSigFile);
if (fseeko(fpSigFile, 0, SEEK_SET)) {
fprintf(stderr, "ERROR: Could not seek to start of sig file.\n");
goto failure;
}
/* Read in the base64 encoded signature to import */
passedInSignatureB64 = malloc(sizeOfBase64EncodedFile + 1);
passedInSignatureB64[sizeOfBase64EncodedFile] = '\0';
if (fread(passedInSignatureB64, sizeOfBase64EncodedFile, 1, fpSigFile) != 1) {
fprintf(stderr, "ERROR: Could read b64 sig file.\n");
goto failure;
}
/* Decode the base64 encoded data */
passedInSignatureRaw = ATOB_AsciiToData(passedInSignatureB64, &passedInSignatureLenRaw);
if (!passedInSignatureRaw) {
fprintf(stderr, "ERROR: could not obtain base64 decoded data\n");
goto failure;
}
/* Read everything up until the signature block offset and write it out */
if (ReadAndWrite(fpSrc, fpDest, buf,
SIGNATURE_BLOCK_OFFSET, "signature block offset")) {
goto failure;
}
/* Get the number of signatures */
if (ReadAndWrite(fpSrc, fpDest, &signatureCount,
sizeof(signatureCount), "signature count")) {
goto failure;
}
signatureCount = ntohl(signatureCount);
if (signatureCount > MAX_SIGNATURES) {
fprintf(stderr, "ERROR: Signature count was out of range\n");
goto failure;
}
if (sigIndex >= signatureCount) {
fprintf(stderr, "ERROR: Signature index was out of range\n");
goto failure;
}
/* Read and write the whole signature block, but if we reach the
signature offset, then we should replace it with the specified
base64 decoded signature */
for (i = 0; i < signatureCount; i++) {
/* Read/Write the signature algorithm ID */
if (ReadAndWrite(fpSrc, fpDest,
&signatureAlgorithmID,
sizeof(signatureAlgorithmID), "sig algorithm ID")) {
goto failure;
}
/* Read/Write the signature length */
if (ReadAndWrite(fpSrc, fpDest,
&signatureLen, sizeof(signatureLen), "sig length")) {
goto failure;
}
signatureLen = ntohl(signatureLen);
/* Get the signature */
if (extractedMARSignature) {
free(extractedMARSignature);
}
extractedMARSignature = malloc(signatureLen);
if (sigIndex == i) {
if (passedInSignatureLenRaw != signatureLen) {
fprintf(stderr, "ERROR: Signature length must be the same\n");
goto failure;
}
if (fread(extractedMARSignature, signatureLen, 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could not read signature\n");
goto failure;
}
if (fwrite(passedInSignatureRaw, passedInSignatureLenRaw,
1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write signature\n");
goto failure;
}
} else {
if (ReadAndWrite(fpSrc, fpDest,
extractedMARSignature, signatureLen, "signature")) {
goto failure;
}
}
}
/* We replaced the signature so let's just skip past the rest o the
file. */
numChunks = (sizeOfSrcMAR - ftello(fpSrc)) / BLOCKSIZE;
leftOver = (sizeOfSrcMAR - ftello(fpSrc)) % BLOCKSIZE;
/* Read each file and write it to the MAR file */
for (i = 0; i < numChunks; ++i) {
if (ReadAndWrite(fpSrc, fpDest, buf, BLOCKSIZE, "content block")) {
goto failure;
}
}
if (ReadAndWrite(fpSrc, fpDest, buf, leftOver, "left over content block")) {
goto failure;
}
rv = 0;
failure:
if (fpSrc) {
fclose(fpSrc);
}
if (fpDest) {
fclose(fpDest);
}
if (fpSigFile) {
fclose(fpSigFile);
}
if (rv) {
remove(dest);
}
if (extractedMARSignature) {
free(extractedMARSignature);
}
if (passedInSignatureB64) {
free(passedInSignatureB64);
}
if (passedInSignatureRaw) {
PORT_Free(passedInSignatureRaw);
}
return rv;
}
// lastfield = 0 for no limit.
static il* solvedfile_getall_val(char* fn, int firstfield, int lastfield, int maxfields, int val) {
FILE* f;
off_t end;
int fields = 0;
il* list;
int i;
unsigned char* map;
list = il_new(256);
f = fopen(fn, "rb");
if (!f) {
// if file doesn't exist, assume no fields are solved.
if (val == 0) {
for (i=firstfield; i<=lastfield; i++) {
il_append(list, i);
fields++;
if (fields == maxfields)
break;
}
}
return list;
}
if (fseek(f, 0, SEEK_END) ||
((end = ftello(f)) == -1)) {
fprintf(stderr, "Error: seeking to end of file %s: %s\n",
fn, strerror(errno));
fclose(f);
il_free(list);
return NULL;
}
// 1-index
firstfield--;
lastfield--;
if (end <= firstfield) {
fclose(f);
return list;
}
map = mmap(NULL, end, PROT_READ, MAP_SHARED, fileno(f), 0);
fclose(f);
if (map == MAP_FAILED) {
fprintf(stderr, "Error: couldn't mmap file %s: %s\n", fn, strerror(errno));
il_free(list);
return NULL;
}
for (i=firstfield; ((lastfield == -1) || (i<=lastfield)) && (i < end); i++) {
if (map[i] == val) {
// 1-index
il_append(list, i+1);
if (il_size(list) == maxfields)
break;
}
}
munmap(map, end);
if (val == 0) {
// fields larger than the file size are unsolved.
for (i=end; i<=lastfield; i++) {
if (il_size(list) == maxfields)
break;
// 1-index
il_append(list, i+1);
}
}
return list;
}
/**
* Writes out a copy of the MAR at src but with embedded signatures.
* The passed in MAR file must not already be signed or an error will
* be returned.
*
* @param NSSConfigDir The NSS directory containing the private key for signing
* @param certNames The nicknames of the certificate to use for signing
* @param certCount The number of certificate names contained in certNames.
* One signature will be produced for each certificate.
* @param src The path of the source MAR file to sign
* @param dest The path of the MAR file to write out that is signed
* @return 0 on success
* -1 on error
*/
int
mar_repackage_and_sign(const char *NSSConfigDir,
const char * const *certNames,
uint32_t certCount,
const char *src,
const char *dest)
{
uint32_t offsetToIndex, dstOffsetToIndex, indexLength,
numSignatures = 0, leftOver,
signatureAlgorithmID, signatureSectionLength = 0;
uint32_t signatureLengths[MAX_SIGNATURES];
int64_t oldPos, sizeOfEntireMAR = 0, realSizeOfSrcMAR,
signaturePlaceholderOffset, numBytesToCopy,
numChunks, i;
FILE *fpSrc = NULL, *fpDest = NULL;
int rv = -1, hasSignatureBlock;
SGNContext *ctxs[MAX_SIGNATURES];
SECItem secItems[MAX_SIGNATURES];
char buf[BLOCKSIZE];
SECKEYPrivateKey *privKeys[MAX_SIGNATURES];
CERTCertificate *certs[MAX_SIGNATURES];
char *indexBuf = NULL;
uint32_t k;
memset(signatureLengths, 0, sizeof(signatureLengths));
memset(ctxs, 0, sizeof(ctxs));
memset(secItems, 0, sizeof(secItems));
memset(privKeys, 0, sizeof(privKeys));
memset(certs, 0, sizeof(certs));
if (!NSSConfigDir || !certNames || certCount == 0 || !src || !dest) {
fprintf(stderr, "ERROR: Invalid parameter passed in.\n");
return -1;
}
if (NSSInitCryptoContext(NSSConfigDir)) {
fprintf(stderr, "ERROR: Could not init config dir: %s\n", NSSConfigDir);
goto failure;
}
PK11_SetPasswordFunc(SECU_GetModulePassword);
fpSrc = fopen(src, "rb");
if (!fpSrc) {
fprintf(stderr, "ERROR: could not open source file: %s\n", src);
goto failure;
}
fpDest = fopen(dest, "wb");
if (!fpDest) {
fprintf(stderr, "ERROR: could not create target file: %s\n", dest);
goto failure;
}
/* Determine if the source MAR file has the new fields for signing or not */
if (get_mar_file_info(src, &hasSignatureBlock, NULL, NULL, NULL, NULL)) {
fprintf(stderr, "ERROR: could not determine if MAR is old or new.\n");
goto failure;
}
for (k = 0; k < certCount; k++) {
if (NSSSignBegin(certNames[k], &ctxs[k], &privKeys[k],
&certs[k], &signatureLengths[k])) {
fprintf(stderr, "ERROR: NSSSignBegin failed\n");
goto failure;
}
}
/* MAR ID */
if (ReadWriteAndUpdateSignatures(fpSrc, fpDest,
buf, MAR_ID_SIZE,
ctxs, certCount, "MAR ID")) {
goto failure;
}
/* Offset to index */
if (fread(&offsetToIndex, sizeof(offsetToIndex), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could not read offset\n");
goto failure;
}
offsetToIndex = ntohl(offsetToIndex);
/* Get the real size of the MAR */
oldPos = ftello(fpSrc);
if (fseeko(fpSrc, 0, SEEK_END)) {
fprintf(stderr, "ERROR: Could not seek to end of file.\n");
goto failure;
}
realSizeOfSrcMAR = ftello(fpSrc);
if (fseeko(fpSrc, oldPos, SEEK_SET)) {
fprintf(stderr, "ERROR: Could not seek back to current location.\n");
goto failure;
}
if (hasSignatureBlock) {
/* Get the MAR length and adjust its size */
if (fread(&sizeOfEntireMAR,
sizeof(sizeOfEntireMAR), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could read mar size\n");
goto failure;
}
sizeOfEntireMAR = NETWORK_TO_HOST64(sizeOfEntireMAR);
if (sizeOfEntireMAR != realSizeOfSrcMAR) {
fprintf(stderr, "ERROR: Source MAR is not of the right size\n");
goto failure;
}
/* Get the num signatures in the source file */
if (fread(&numSignatures, sizeof(numSignatures), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could read num signatures\n");
goto failure;
}
numSignatures = ntohl(numSignatures);
/* We do not support resigning, if you have multiple signatures,
you must add them all at the same time. */
if (numSignatures) {
fprintf(stderr, "ERROR: MAR is already signed\n");
goto failure;
}
} else {
sizeOfEntireMAR = realSizeOfSrcMAR;
}
if (((int64_t)offsetToIndex) > sizeOfEntireMAR) {
fprintf(stderr, "ERROR: Offset to index is larger than the file size.\n");
goto failure;
}
/* Calculate the total signature block length */
for (k = 0; k < certCount; k++) {
signatureSectionLength += sizeof(signatureAlgorithmID) +
sizeof(signatureLengths[k]) +
signatureLengths[k];
}
dstOffsetToIndex = offsetToIndex;
if (!hasSignatureBlock) {
dstOffsetToIndex += sizeof(sizeOfEntireMAR) + sizeof(numSignatures);
}
dstOffsetToIndex += signatureSectionLength;
/* Write out the index offset */
dstOffsetToIndex = htonl(dstOffsetToIndex);
if (WriteAndUpdateSignatures(fpDest, &dstOffsetToIndex,
sizeof(dstOffsetToIndex), ctxs, certCount,
"index offset")) {
goto failure;
}
dstOffsetToIndex = ntohl(dstOffsetToIndex);
/* Write out the new MAR file size */
sizeOfEntireMAR += signatureSectionLength;
if (!hasSignatureBlock) {
sizeOfEntireMAR += sizeof(sizeOfEntireMAR) + sizeof(numSignatures);
}
/* Write out the MAR size */
sizeOfEntireMAR = HOST_TO_NETWORK64(sizeOfEntireMAR);
if (WriteAndUpdateSignatures(fpDest, &sizeOfEntireMAR,
sizeof(sizeOfEntireMAR), ctxs, certCount,
"size of MAR")) {
goto failure;
}
sizeOfEntireMAR = NETWORK_TO_HOST64(sizeOfEntireMAR);
/* Write out the number of signatures */
numSignatures = certCount;
numSignatures = htonl(numSignatures);
if (WriteAndUpdateSignatures(fpDest, &numSignatures,
sizeof(numSignatures), ctxs, certCount,
"num signatures")) {
goto failure;
}
numSignatures = ntohl(numSignatures);
signaturePlaceholderOffset = ftello(fpDest);
for (k = 0; k < certCount; k++) {
/* Write out the signature algorithm ID, Only an ID of 1 is supported */
signatureAlgorithmID = htonl(1);
if (WriteAndUpdateSignatures(fpDest, &signatureAlgorithmID,
sizeof(signatureAlgorithmID),
ctxs, certCount, "num signatures")) {
goto failure;
}
signatureAlgorithmID = ntohl(signatureAlgorithmID);
/* Write out the signature length */
signatureLengths[k] = htonl(signatureLengths[k]);
if (WriteAndUpdateSignatures(fpDest, &signatureLengths[k],
sizeof(signatureLengths[k]),
ctxs, certCount, "signature length")) {
goto failure;
}
signatureLengths[k] = ntohl(signatureLengths[k]);
/* Write out a placeholder for the signature, we'll come back to this later
*** THIS IS NOT SIGNED because it is a placeholder that will be replaced
below, plus it is going to be the signature itself. *** */
memset(buf, 0, sizeof(buf));
if (fwrite(buf, signatureLengths[k], 1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write signature length\n");
goto failure;
}
}
/* Write out the rest of the MAR excluding the index header and index
offsetToIndex unfortunately has to remain 32-bit because for backwards
compatibility with the old MAR file format. */
if (ftello(fpSrc) > ((int64_t)offsetToIndex)) {
fprintf(stderr, "ERROR: Index offset is too small.\n");
goto failure;
}
numBytesToCopy = ((int64_t)offsetToIndex) - ftello(fpSrc);
numChunks = numBytesToCopy / BLOCKSIZE;
leftOver = numBytesToCopy % BLOCKSIZE;
/* Read each file and write it to the MAR file */
for (i = 0; i < numChunks; ++i) {
if (ReadWriteAndUpdateSignatures(fpSrc, fpDest, buf,
BLOCKSIZE, ctxs, certCount,
"content block")) {
goto failure;
}
}
/* Write out the left over */
if (ReadWriteAndUpdateSignatures(fpSrc, fpDest, buf,
leftOver, ctxs, certCount,
"left over content block")) {
goto failure;
}
/* Length of the index */
if (ReadWriteAndUpdateSignatures(fpSrc, fpDest, &indexLength,
sizeof(indexLength), ctxs, certCount,
"index length")) {
goto failure;
}
indexLength = ntohl(indexLength);
/* Consume the index and adjust each index by signatureSectionLength */
indexBuf = malloc(indexLength);
if (fread(indexBuf, indexLength, 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could not read index\n");
goto failure;
}
/* Adjust each entry in the index */
if (hasSignatureBlock) {
AdjustIndexContentOffsets(indexBuf, indexLength, signatureSectionLength);
} else {
AdjustIndexContentOffsets(indexBuf, indexLength,
sizeof(sizeOfEntireMAR) +
sizeof(numSignatures) +
signatureSectionLength);
}
if (WriteAndUpdateSignatures(fpDest, indexBuf,
indexLength, ctxs, certCount, "index")) {
goto failure;
}
/* Ensure that we don't sign a file that is too large to be accepted by
the verification function. */
if (ftello(fpDest) > MAX_SIZE_OF_MAR_FILE) {
goto failure;
}
for (k = 0; k < certCount; k++) {
/* Get the signature */
if (SGN_End(ctxs[k], &secItems[k]) != SECSuccess) {
fprintf(stderr, "ERROR: Could not end signature context\n");
goto failure;
}
if (signatureLengths[k] != secItems[k].len) {
fprintf(stderr, "ERROR: Signature is not the expected length\n");
goto failure;
}
}
/* Get back to the location of the signature placeholder */
if (fseeko(fpDest, signaturePlaceholderOffset, SEEK_SET)) {
fprintf(stderr, "ERROR: Could not seek to signature offset\n");
goto failure;
}
for (k = 0; k < certCount; k++) {
/* Skip to the position of the next signature */
if (fseeko(fpDest, sizeof(signatureAlgorithmID) +
sizeof(signatureLengths[k]), SEEK_CUR)) {
fprintf(stderr, "ERROR: Could not seek to signature offset\n");
goto failure;
}
/* Write out the calculated signature.
*** THIS IS NOT SIGNED because it is the signature itself. *** */
if (fwrite(secItems[k].data, secItems[k].len, 1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write signature\n");
goto failure;
}
}
rv = 0;
failure:
if (fpSrc) {
fclose(fpSrc);
}
if (fpDest) {
fclose(fpDest);
}
if (rv) {
remove(dest);
}
if (indexBuf) {
free(indexBuf);
}
/* Cleanup */
for (k = 0; k < certCount; k++) {
if (ctxs[k]) {
SGN_DestroyContext(ctxs[k], PR_TRUE);
}
if (certs[k]) {
CERT_DestroyCertificate(certs[k]);
}
if (privKeys[k]) {
SECKEY_DestroyPrivateKey(privKeys[k]);
}
SECITEM_FreeItem(&secItems[k], PR_FALSE);
}
if (rv) {
remove(dest);
}
return rv;
}
int main(int argc, char *argv[])
{
FILE *input;
int opt;
off_t pos;
size_t size;
uint32_t addr = 0x40200000;
uint32_t temp;
int chsettings = 0;
int swap = 0;
while((opt = getopt(argc, argv, "a:s")) != -1) {
switch (opt) {
case 'a':
addr = strtoul(optarg, NULL, 0);
break;
case 's':
swap = 1;
break;
}
}
if (optind >= argc) {
usage(argv[0]);
exit(1);
}
input = fopen(argv[optind], "r");
if (input == NULL) {
perror("fopen");
exit(EXIT_FAILURE);
}
if (fseeko(input, 0, SEEK_END) == -1) {
perror("fseeko");
exit(EXIT_FAILURE);
}
pos = ftello(input);
if (pos == -1) {
perror("ftello");
exit(EXIT_FAILURE);
}
if (pos > 0x100000) {
fprintf(stderr, "error: image should be smaller than 1 MiB\n");
exit(EXIT_FAILURE);
}
if (fseeko(input, 0x14, SEEK_SET) == -1) {
perror("fseeko");
exit(EXIT_FAILURE);
}
size = fread(&temp, 1, sizeof(uint32_t), input);
if (!size) {
perror("fseeko");
exit(EXIT_FAILURE);
}
/*
* Test if this is an image generated with omap_signGP. These don't
* need size and load address prepended.
*/
if (le32toh(temp) == 0x45534843)
chsettings = 1;
if (fseeko(input, 0x0, SEEK_SET) == -1) {
perror("fseeko");
exit(EXIT_FAILURE);
}
pos = (pos + 3) & ~3;
if (!chsettings) {
/* image size */
temp = pos;
if (swap)
temp = htobe32(temp);
fwrite(&temp, sizeof(uint32_t), 1, stdout);
/* memory address */
temp = addr;
if (swap)
temp = htobe32(temp);
fwrite(&temp, sizeof(uint32_t), 1, stdout);
}
for (;;) {
size = fread(&temp, 1, sizeof(uint32_t), input);
if (!size)
break;
if (size < 4 && !feof(input)) {
perror("fread");
exit(EXIT_FAILURE);
}
if (swap)
temp = htobe32(le32toh(temp));
if (fwrite(&temp, 1, sizeof(uint32_t), stdout) != 4) {
perror("fwrite");
exit(EXIT_FAILURE);
}
}
if (fclose(input) != 0) {
perror("fclose");
exit(EXIT_FAILURE);
}
exit(EXIT_SUCCESS);
}
int64 FileStream::tell(void)
{
return ftello(fp);
}
WriterPlaceholder<T> writePlaceholder() {
off_t position = ftello(out);
write((T)0);
return WriterPlaceholder<T>(*this, position);
}
ZIP_EXTERN int
zip_close(struct zip *za)
{
int survivors;
int i, j, error;
char *temp;
FILE *out;
mode_t mask;
struct zip_cdir *cd;
struct zip_dirent de;
struct filelist *filelist;
int reopen_on_error;
int new_torrentzip;
reopen_on_error = 0;
if (za == NULL)
return -1;
if (!_zip_changed(za, &survivors)) {
_zip_free(za);
return 0;
}
/* don't create zip files with no entries */
if (survivors == 0) {
if (za->zn && za->zp) {
if (remove(za->zn) != 0) {
_zip_error_set(&za->error, ZIP_ER_REMOVE, errno);
return -1;
}
}
_zip_free(za);
return 0;
}
if ((filelist=(struct filelist *)malloc(sizeof(filelist[0])*survivors))
== NULL)
return -1;
if ((cd=_zip_cdir_new(survivors, &za->error)) == NULL) {
free(filelist);
return -1;
}
for (i=0; i<survivors; i++)
_zip_dirent_init(&cd->entry[i]);
/* archive comment is special for torrentzip */
if (zip_get_archive_flag(za, ZIP_AFL_TORRENT, 0)) {
cd->comment = _zip_memdup(TORRENT_SIG "XXXXXXXX",
TORRENT_SIG_LEN + TORRENT_CRC_LEN,
&za->error);
if (cd->comment == NULL) {
_zip_cdir_free(cd);
free(filelist);
return -1;
}
cd->comment_len = TORRENT_SIG_LEN + TORRENT_CRC_LEN;
}
else if (zip_get_archive_flag(za, ZIP_AFL_TORRENT, ZIP_FL_UNCHANGED) == 0) {
if (_zip_cdir_set_comment(cd, za) == -1) {
_zip_cdir_free(cd);
free(filelist);
return -1;
}
}
if ((temp=_zip_create_temp_output(za, &out)) == NULL) {
_zip_cdir_free(cd);
free(filelist);
return -1;
}
/* create list of files with index into original archive */
for (i=j=0; i<za->nentry; i++) {
if (za->entry[i].state == ZIP_ST_DELETED)
continue;
filelist[j].idx = i;
filelist[j].name = zip_get_name(za, i, 0);
j++;
}
if (zip_get_archive_flag(za, ZIP_AFL_TORRENT, 0))
qsort(filelist, survivors, sizeof(filelist[0]),
_zip_torrentzip_cmp);
new_torrentzip = (zip_get_archive_flag(za, ZIP_AFL_TORRENT, 0) == 1
&& zip_get_archive_flag(za, ZIP_AFL_TORRENT,
ZIP_FL_UNCHANGED) == 0);
error = 0;
for (j=0; j<survivors; j++) {
i = filelist[j].idx;
/* create new local directory entry */
if (ZIP_ENTRY_DATA_CHANGED(za->entry+i) || new_torrentzip) {
_zip_dirent_init(&de);
if (zip_get_archive_flag(za, ZIP_AFL_TORRENT, 0))
_zip_dirent_torrent_normalize(&de);
/* use it as central directory entry */
memcpy(cd->entry+j, &de, sizeof(cd->entry[j]));
/* set/update file name */
if (za->entry[i].ch_filename == NULL) {
if (za->entry[i].state == ZIP_ST_ADDED) {
de.filename = strdup("-");
de.filename_len = 1;
cd->entry[j].filename = "-";
cd->entry[j].filename_len = 1;
}
else {
de.filename = strdup(za->cdir->entry[i].filename);
de.filename_len = strlen(de.filename);
cd->entry[j].filename = za->cdir->entry[i].filename;
cd->entry[j].filename_len = de.filename_len;
}
}
}
else {
/* copy existing directory entries */
if (fseeko(za->zp, za->cdir->entry[i].offset, SEEK_SET) != 0) {
_zip_error_set(&za->error, ZIP_ER_SEEK, errno);
error = 1;
break;
}
if (_zip_dirent_read(&de, za->zp, NULL, NULL, 1,
&za->error) != 0) {
error = 1;
break;
}
memcpy(cd->entry+j, za->cdir->entry+i, sizeof(cd->entry[j]));
if (de.bitflags & ZIP_GPBF_DATA_DESCRIPTOR) {
de.crc = za->cdir->entry[i].crc;
de.comp_size = za->cdir->entry[i].comp_size;
de.uncomp_size = za->cdir->entry[i].uncomp_size;
de.bitflags &= ~ZIP_GPBF_DATA_DESCRIPTOR;
cd->entry[j].bitflags &= ~ZIP_GPBF_DATA_DESCRIPTOR;
}
}
if (za->entry[i].ch_filename) {
free(de.filename);
if ((de.filename=strdup(za->entry[i].ch_filename)) == NULL) {
error = 1;
break;
}
de.filename_len = strlen(de.filename);
cd->entry[j].filename = za->entry[i].ch_filename;
cd->entry[j].filename_len = de.filename_len;
}
if (zip_get_archive_flag(za, ZIP_AFL_TORRENT, 0) == 0
&& za->entry[i].ch_comment_len != -1) {
/* as the rest of cd entries, its malloc/free is done by za */
cd->entry[j].comment = za->entry[i].ch_comment;
cd->entry[j].comment_len = za->entry[i].ch_comment_len;
}
cd->entry[j].offset = ftello(out);
if (ZIP_ENTRY_DATA_CHANGED(za->entry+i) || new_torrentzip) {
struct zip_source *zs;
zs = NULL;
if (!ZIP_ENTRY_DATA_CHANGED(za->entry+i)) {
if ((zs=zip_source_zip(za, za, i, ZIP_FL_RECOMPRESS, 0, -1))
== NULL) {
error = 1;
break;
}
}
if (add_data(za, zs ? zs : za->entry[i].source, &de, out) < 0) {
error = 1;
break;
}
cd->entry[j].last_mod = de.last_mod;
cd->entry[j].comp_method = de.comp_method;
cd->entry[j].comp_size = de.comp_size;
cd->entry[j].uncomp_size = de.uncomp_size;
cd->entry[j].crc = de.crc;
}
else {
if (_zip_dirent_write(&de, out, 1, &za->error) < 0) {
error = 1;
break;
}
/* we just read the local dirent, file is at correct position */
if (copy_data(za->zp, cd->entry[j].comp_size, out,
&za->error) < 0) {
error = 1;
break;
}
}
_zip_dirent_finalize(&de);
}
free(filelist);
if (!error) {
if (write_cdir(za, cd, out) < 0)
error = 1;
}
/* pointers in cd entries are owned by za */
cd->nentry = 0;
_zip_cdir_free(cd);
if (error) {
_zip_dirent_finalize(&de);
fclose(out);
remove(temp);
free(temp);
return -1;
}
if (fclose(out) != 0) {
_zip_error_set(&za->error, ZIP_ER_CLOSE, errno);
remove(temp);
free(temp);
return -1;
}
if (za->zp) {
fclose(za->zp);
za->zp = NULL;
reopen_on_error = 1;
}
if (_zip_rename(temp, za->zn) != 0) {
_zip_error_set(&za->error, ZIP_ER_RENAME, errno);
remove(temp);
free(temp);
if (reopen_on_error) {
/* ignore errors, since we're already in an error case */
za->zp = fopen(za->zn, "rb");
}
return -1;
}
mask = umask(0);
umask(mask);
chmod(za->zn, 0666&~mask);
_zip_free(za);
free(temp);
return 0;
}
/** Writes a coin with the given value */
void write(T value) {
off_t oldPosition = ftello(writer.out);
fseek(writer.out, position, SEEK_SET);
writer.write(value);
fseek(writer.out, oldPosition, SEEK_SET);
}
int mcfioC_NextEvent(int stream)
/*
** The Core routine for getting or setting the next event d.s. from/to
** a stream.
**
*/
{
int jstr, idtmp, ntot, nn1;
off_t p_evt, p2;
mcfStream *str;
if (McfStreamPtrList == NULL) {
fprintf(stderr,
" mcfio_NextEvent: You must first initialize by calling mcfio_Init.\n");
return -1;
}
jstr = stream-1;
if (McfStreamPtrList[jstr] == NULL) {
fprintf(stderr,
" mcfio_NextEvent: First, declare the stream by calling mcfio_Open...\n");
return -1;
}
str = McfStreamPtrList[jstr];
if (str->dos == MCFIO_SEQUENTIAL) return mcfioC_NextEventSequential(stream);
if (str->row == MCFIO_READ) {
/*
** Read the next event, hunt for either an event or a table of event
** if event table not available.
*/
if ((str->table == NULL) ||
((str->table != NULL)&& (str->table->evtnums == NULL))) {
idtmp = mcfioC_gofornextevent(str);
if (idtmp != EVENTTABLE) {
if (str->table !=NULL)
mcfioC_Free_EventTable(&(str->table));
if (idtmp == NOTHING) return -1;
p_evt = str->currentPos;
} else {
if( xdr_mcfast_eventtable(str->xdr, &idtmp,
&ntot, McfGenericVersion, &(str->table)) == FALSE) {
fprintf(stderr,
" mcfio_NextEvent: XDR Error decoding the EventTable \n");
return -1;
}
p2 = ftello(str->filePtr);
str->numWordsC += (ntot/4);
str->numWordsT += ((p2-str->currentPos)/4);
str->currentPos = p2;
str->table->ievt = 0;
/*
** If table empty, cal this routine recursively to get
** the next event
*/
if (str->table->numevts <= 0) {
if (str->table->nextLocator == -1)
mcfioC_Free_EventTable(&(str->table));
return mcfioC_NextEvent(str->id);
}
p_evt = str->table->ptrEvents[0];
}
} else {
if (str->table->ievt < str->table->numevts) {
p_evt = str->table->ptrEvents[str->table->ievt];
} else {
/*
** decode the next table, if valid. If not, scrap the
** existing table and call next event recursively.
*/
if (str->table->nextLocator == -2) {
/*
** Stream is at EOF
*/
str->status = MCFIO_EOF;
return MCFIO_EOF;
} else if (str->table->nextLocator == -1) {
fprintf(stderr,
" mcfio_NextEvent: Corrupted Event Table \n");
return -1;
}
if (fseeko(str->filePtr,str->table->nextLocator,SEEK_SET) != 0) {
fprintf(stderr,
" mcfio_NextEvent: Error Repositioning stream \n");
return -1;
}
if( xdr_mcfast_eventtable(str->xdr, &idtmp,
&ntot, McfGenericVersion, &(str->table)) == FALSE) {
fprintf(stderr,
" mcfio_NextEvent: XDR Error decoding the EventTable \n");
return -1;
}
p2 = ftello(str->filePtr);
str->numWordsC += (ntot/4);
str->numWordsT += ((p2-str->currentPos)/4);
str->currentPos = p2;
str->table->ievt = 0;
p_evt = str->table->ptrEvents[0];
}
}
/*
** we should be pointing to a good event header here.
*/
if (fseeko(str->filePtr,p_evt,SEEK_SET) != 0) return -1;
if( xdr_mcfast_eventheader(str->xdr, &idtmp,
&ntot, McfGenericVersion, &(str->ehead)) == FALSE) return -1;
str->currentPos = ftello(str->filePtr);
str->numWordsC += (ntot/4);
str->numWordsT += ((str->currentPos - p_evt)/4);
if (str->table != NULL) str->table->ievt ++;
return MCFIO_RUNNING;
} else {
/*
** Writing Code here.
*/
str->table->numevts++;
str->fhead->numevts++;
if (str->ehead->previousevtnum == str->ehead->evtnum) str->ehead->evtnum++;
/*
** Write the current event header, normal case. First Flush the current
** event, then initiate the next one event. Note that wrtevt will
** reposition the stream after rewriting the event header, if FLUSH.
** e.g. ready to initiate either a new table or a new event.
*/
if (mcfioC_WrtEvt(str, FLUSH) == FALSE) return -1;
str->ehead->previousevtnum = str->ehead->evtnum;
if (str->table->numevts == (str->fhead->dimTable - 1)) {
/*
** The Event table is now full. Flush it. Then initiate a new table.
*/
str->table->nextLocator = ftello(str->filePtr);
if (mcfioC_Wrttable(str, FLUSH) == FALSE) return -1;
if (mcfioC_Wrttable(str, INITIATE) == FALSE) return -1;
}
str->ehead->nBlocks = 0;
str->ehead->nNTuples = 0;
nn1 = str->ehead->evtnum;
if (mcfioC_WrtEvt(str, INITIATE) == FALSE) return -1;
str->ehead->evtnum = nn1;
return MCFIO_RUNNING;
}
}
static bool
read_num (FILE *fp, char const *fieldname,
intmax_t min_val, uintmax_t max_val, intmax_t *pval)
{
int i;
char buf[INT_BUFSIZE_BOUND (intmax_t)];
char offbuf[INT_BUFSIZE_BOUND (off_t)];
char minbuf[INT_BUFSIZE_BOUND (intmax_t)];
char maxbuf[INT_BUFSIZE_BOUND (intmax_t)];
int conversion_errno;
int c = getc (fp);
bool negative = c == '-';
for (i = 0; (i == 0 && negative) || ISDIGIT (c); i++)
{
buf[i] = c;
if (i == sizeof buf - 1)
FATAL_ERROR ((0, 0,
_("%s: byte %s: %s %.*s... too long"),
quotearg_colon (listed_incremental_option),
offtostr (ftello (fp), offbuf),
fieldname, i + 1, buf));
c = getc (fp);
}
buf[i] = 0;
if (c < 0)
{
if (ferror (fp))
read_fatal (listed_incremental_option);
if (i != 0)
FATAL_ERROR ((0, 0, "%s: %s",
quotearg_colon (listed_incremental_option),
_("Unexpected EOF in snapshot file")));
return false;
}
if (c)
{
unsigned uc = c;
FATAL_ERROR ((0, 0,
_("%s: byte %s: %s %s followed by invalid byte 0x%02x"),
quotearg_colon (listed_incremental_option),
offtostr (ftello (fp), offbuf),
fieldname, buf, uc));
}
*pval = strtosysint (buf, NULL, min_val, max_val);
conversion_errno = errno;
switch (conversion_errno)
{
case ERANGE:
FATAL_ERROR ((0, conversion_errno,
_("%s: byte %s: (valid range %s..%s)\n\t%s %s"),
quotearg_colon (listed_incremental_option),
offtostr (ftello (fp), offbuf),
imaxtostr (min_val, minbuf),
umaxtostr (max_val, maxbuf), fieldname, buf));
default:
FATAL_ERROR ((0, conversion_errno,
_("%s: byte %s: %s %s"),
quotearg_colon (listed_incremental_option),
offtostr (ftello (fp), offbuf), fieldname, buf));
case 0:
break;
}
return true;
}
uint8_t asfHeader::loadVideo(asfChunk *s)
{
uint32_t w,h,x;
printf("--\n");
w=s->read32();
h=s->read32();
s->read8();
x=s->read16();
_isvideopresent=1;
memset(&_mainaviheader,0,sizeof(_mainaviheader));
_mainaviheader.dwWidth=w;
_mainaviheader.dwHeight=h;
_video_bih.biWidth=w;
_video_bih.biHeight=h;
printf("Pic Width %04d\n",w);
printf("Pic Height %04d\n",h);
printf(" BMP size %04d (%04d)\n",x,(int)sizeof(ADM_BITMAPINFOHEADER));
s->read((uint8_t *)&_video_bih,sizeof(ADM_BITMAPINFOHEADER));
#ifdef ADM_BIG_ENDIAN
Endian_BitMapInfo(&_video_bih);
#endif
_videostream.fccHandler=_video_bih.biCompression;
printf("Codec : <%s> (%04x)\n",
fourCC::tostring(_video_bih.biCompression),_video_bih.biCompression);
if(fourCC::check(_video_bih.biCompression,(uint8_t *)"DVR "))
{
// It is MS DVR, fail so that the mpeg2 indexer can take it from here
_videostream.fccHandler=_video_bih.biCompression=fourCC::get((uint8_t *)"MPEG");
printf("This is MSDVR, not ASF\n");
return 0;
}
printBih(&_video_bih);
#if 1
if(_video_bih.biSize>sizeof(ADM_BITMAPINFOHEADER))
{
x=_video_bih.biSize;
#else
if(x>sizeof(ADM_BITMAPINFOHEADER))
{
#endif
_videoExtraLen=x-sizeof(ADM_BITMAPINFOHEADER);
_videoExtraData=new uint8_t[_videoExtraLen];
s->read(_videoExtraData,_videoExtraLen);
ADM_info("We have %d bytes of extra data for video.\n",(int)_videoExtraLen);
}else
{
ADM_info("No extra data for video\n");
}
uint64_t l=ftello(_fd);
printf("Bytes left : %d\n",(int)(s->endPos()-l));
return 1;
}
/**
\fn buildIndex
\brief Scan the file to build an index
Header Chunk
Chunk
Chunk
Chunk
Data chunk
Chunk
Chunk
We skip the 1st one, and just read the header of the 2nd one
*/
uint8_t asfHeader::buildIndex(void)
{
uint32_t fSize;
const chunky *id;
uint32_t chunkFound;
uint32_t r=5;
uint32_t len;
fseeko(_fd,0,SEEK_END);
fSize=ftello(_fd);
fseeko(_fd,0,SEEK_SET);
asfChunk h(_fd);
printf("[ASF] ********** Building index **********\n");
printf("[ASF] Searching data\n");
while(r--)
{
h.nextChunk(); // Skip headers
id=h.chunkId();
h.dump();
if(id->id==ADM_CHUNK_DATA_CHUNK) break;
h.skipChunk();
}
if(id->id!=ADM_CHUNK_DATA_CHUNK) return 0;
// Remove leftover from DATA_chunk
// Unknown GUID 16
// Number of packets UINT64 8
// Unknown UINT8 1
// Unknown UINT8 1
//
h.read32();
h.read32();
h.read32();
h.read32();
_nbPackets=(uint32_t) h.read64();
h.read16();
len=h.chunkLen-16-8-2-24;
printf("[ASF] nbPacket : %u\n",_nbPackets);
printf("[ASF] len to go : %u\n",len);
printf("[ASF] scanning data\n");
_dataStartOffset=ftello(_fd);
// Here we go
asfPacket *aPacket=new asfPacket(_fd,_nbPackets,_packetSize,
&readQueue,&storageQueue,_dataStartOffset);
uint32_t packet=0;
#define MAXIMAGE (_nbPackets)
uint32_t sequence=0;
uint32_t ceilImage=MAXIMAGE;
nbImage=0;
len=0;
asfIndex indexEntry;
memset(&indexEntry,0,sizeof(indexEntry));
bool first=true;
DIA_workingBase *progressBar=createWorking("Indexing");
uint32_t fileSizeMB=(uint32_t)(fSize>>10);
uint64_t lastDts[ASF_MAX_AUDIO_TRACK];
for(int i=0;i<ASF_MAX_AUDIO_TRACK;i++)
{
lastDts[i]=0;
}
while(packet<_nbPackets)
{
while(readQueue.size())
{
asfBit *bit=NULL;
// update UI
uint32_t curPos=(uint32_t)(ftello(_fd)>>10);
progressBar->update(curPos,fileSizeMB);
bit=readQueue.front();
readQueue.pop_front();
// --
uint64_t dts=bit->dts;
uint64_t pts=bit->pts;
aprintf("** DTS=%s\n",ADM_us2plain(dts));
aprintf("** PDTS=%s\n",ADM_us2plain(pts));
if(bit->stream==_videoStreamId)
{
aprintf(">found video packet of size=%d off=%d seq %d, while curseq =%d, dts=%s",
bit->len,bit->offset, bit->sequence,curSeq, ADM_us2plain(dts));
aprintf(" pts=%s\n",ADM_us2plain(pts));
if(bit->sequence!=sequence || first==true)
{
if(first==false)
{
indexEntry.frameLen=len;
aprintf("Pushing video frame index=%d seq=%d pts=%s \n",
_index.size(),
indexEntry.segNb,ADM_us2plain(indexEntry.pts));
aprintf("dts=%s\n",ADM_us2plain(indexEntry.dts));
_index.append(indexEntry);
}
aprintf("New sequence\n");
if( ((sequence+1)&0xff)!=(bit->sequence&0xff))
{
ADM_warning("!!!!!!!!!!!! non continuous sequence %u %u\n",sequence,bit->sequence);
}
indexEntry.frameLen=0;
indexEntry.segNb=bit->sequence;
indexEntry.packetNb=bit->packet;
indexEntry.flags=bit->flags;
indexEntry.dts=dts;
indexEntry.pts=pts;
if(first==false)
{
sequence=bit->sequence;
readQueue.push_front(bit); // reuse it next time
len=0;
continue;
}else
{
sequence=bit->sequence;
first=false; // first packet
}
}
len+=bit->len;
} // End of video stream Id
else // Audio ?
{
int found=0;
for(int i=0;i<_nbAudioTrack && !found;i++)
{
if(bit->stream == _allAudioTracks[i].streamIndex)
{
if(bit->pts!=ADM_NO_PTS)
{
if(!lastDts[i] || (bit->pts>lastDts[i]+500000L)) // seek point every 500 ms
{
asfAudioSeekPoint seek;
seek.pts=bit->pts;
seek.packetNb=bit->packet;
(audioSeekPoints[i]).append(seek);
#if 1
if(!lastDts[i])
printf("Adding seek point for track %d at %s (packet=%d)\n",
i,ADM_us2plain(bit->pts),(int)seek.packetNb);
#endif
lastDts[i]=bit->pts;
aprintf("Adding seek point for track %d at %s (packet=%d)\n",
i,ADM_us2plain(bit->pts),(int)seek.packetNb);
}
}
found=1;
}
}
if(!found)
{
printf("Unmapped stream %u\n",bit->stream);
}
}
delete [] bit->data;
bit->data=NULL;
storageQueue.push_back(bit);
}
//working->update(packet,_nbPackets);
packet++;
aPacket->nextPacket(0xff); // All packets
aPacket->skipPacket();
}
delete progressBar;
delete aPacket;
//delete working;
/* Compact index */
fseeko(_fd,_dataStartOffset,SEEK_SET);
printf("[ASF] %u images found\n",nbImage);
printf("[ASF] ******** End of buildindex *******\n");
nbImage=_index.size();;
if(!nbImage) return 0;
uint64_t shift=60*1000*1000;
bool canShift=false;
uint64_t tPts;
tPts=_index[0].pts;
ADM_info("First image pts: %s, dts: %s\n",ADM_us2plain(tPts), ADM_us2plain(_index[0].dts));
if(tPts != ADM_NO_PTS)
{
shift=tPts;
ADM_info("Video shift = %s\n",ADM_us2plain(tPts));
canShift=true;
}else
canShift=false;
for(int i=0;i<_nbAudioTrack;i++)
{
if(!audioSeekPoints[i].size())
{
ADM_info("audio track : %d, no seek\n",i);
canShift=false;
continue;
}
tPts=audioSeekPoints[i][0].pts;
ADM_info("audio track : %d, %s\n",i,ADM_us2plain(tPts));
if(tPts<shift) shift=tPts;
}
if(canShift)
{
ADM_info("Shifting a/v raw=%s\n",ADM_us2plain(shift));
}else
{
ADM_info("Can t shift\n");
shift=0;
}
_videostream.dwLength=_mainaviheader.dwTotalFrames=nbImage;
_index[0].flags=AVI_KEY_FRAME;
if(_index[0].pts==ADM_NO_PTS) _index[0].pts=_index[0].dts;
// Update fps
// In fact it is an average fps
//
_videostream.dwScale=1000;
// check if we have a duration per frame for video...
int n=frameDurationMapping.size();
int dex=-1;
for(int i=0;i<n;i++)
{
if(frameDurationMapping[i].streamNb==_videoStreamId)
dex=i;
}
if(dex!=-1)
{
ADM_info("Average fps provided\n");
setFps(frameDurationMapping[dex].usPerFrame);
}
else
{
ADM_info("Fps not provided, guessing it from nbFrame and duration\n");
uint32_t avgFps;
if(_index[nbImage-1].pts!=ADM_NO_PTS && _index[0].pts!=ADM_NO_PTS)
{
float f=(_index[nbImage-1].pts-_index[0].pts);
f/=nbImage; // average duration of 1 image in us
setFps((uint64_t) f);
}else
{
printf("[Asf] No pts, setting 30 fps hardcoded\n");
_videostream.dwRate=(uint32_t)30000;;
}
}
if(shift)
{
double frames3=_videostream.dwScale;
frames3/=_videostream.dwRate;
frames3*=3*1000*1000; //
ADM_info("3 frames time = %s\n",ADM_us2plain((uint64_t)frames3));
uint64_t frame64=(uint64_t)frames3;
if(frame64<shift) shift=shift-frame64;
else shift=0;
shiftAudioVideoBy(shift);
}
return 1;
}
/* Flush all pending data on STREAM according to POSIX rules. Both
output and seekable input streams are supported. */
int
rpl_fflush (FILE *stream)
{
/* When stream is NULL, POSIX and C99 only require flushing of "output
streams and update streams in which the most recent operation was not
input", and all implementations do this.
When stream is "an output stream or an update stream in which the most
recent operation was not input", POSIX and C99 requires that fflush
writes out any buffered data, and all implementations do this.
When stream is, however, an input stream or an update stream in
which the most recent operation was input, C99 specifies nothing,
and POSIX only specifies behavior if the stream is seekable.
mingw, in particular, drops the input buffer, leaving the file
descriptor positioned at the end of the input buffer. I.e. ftell
(stream) is lost. We don't want to call the implementation's
fflush in this case.
We test ! freading (stream) here, rather than fwriting (stream), because
what we need to know is whether the stream holds a "read buffer", and on
mingw this is indicated by _IOREAD, regardless of _IOWRT. */
if (stream == NULL || ! freading (stream))
return fflush (stream);
#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Antares, Linux libc5 */
clear_ungetc_buffer_preserving_position (stream);
return fflush (stream);
#else
{
/* Notes about the file-position indicator:
1) The file position indicator is incremented by fgetc() and decremented
by ungetc():
<http://www.opengroup.org/susv3/functions/fgetc.html>
"... the fgetc() function shall ... advance the associated file
position indicator for the stream ..."
<http://www.opengroup.org/susv3/functions/ungetc.html>
"The file-position indicator is decremented by each successful
call to ungetc()..."
2) <http://www.opengroup.org/susv3/functions/ungetc.html> says:
"The value of the file-position indicator for the stream after
reading or discarding all pushed-back bytes shall be the same
as it was before the bytes were pushed back."
Here we are discarding all pushed-back bytes. But more specifically,
3) <http://www.opengroup.org/austin/aardvark/latest/xshbug3.txt> says:
"[After fflush(),] the file offset of the underlying open file
description shall be set to the file position of the stream, and
any characters pushed back onto the stream by ungetc() ... shall
be discarded." */
/* POSIX does not specify fflush behavior for non-seekable input
streams. Some implementations purge unread data, some return
EBADF, some do nothing. */
off_t pos = ftello (stream);
if (pos == -1)
{
errno = EBADF;
return EOF;
}
/* Clear the ungetc buffer. */
clear_ungetc_buffer (stream);
/* To get here, we must be flushing a seekable input stream, so the
semantics of fpurge are now appropriate to clear the buffer. To
avoid losing data, the lseek is also necessary. */
{
int result = fpurge (stream);
if (result != 0)
return result;
}
# if (defined __sferror || defined __DragonFly__) && defined __SNPT /* FreeBSD, NetBSD, OpenBSD, DragonFly, MacOS X, Cygwin */
{
/* Disable seek optimization for the next fseeko call. This tells the
following fseeko call to seek to the desired position directly, rather
than to seek to a block-aligned boundary. */
int saved_flags = disable_seek_optimization (stream);
int result = fseeko (stream, pos, SEEK_SET);
restore_seek_optimization (stream, saved_flags);
return result;
}
# else
pos = lseek (fileno (stream), pos, SEEK_SET);
if (pos == -1)
return EOF;
/* After a successful lseek, update the file descriptor's position cache
in the stream. */
update_fpos_cache (stream, pos);
return 0;
# endif
}
#endif
}
static int gsm_trunc(struct ast_filestream *fs)
{
return ftruncate(fileno(fs->f), ftello(fs->f));
}
/**
* Writes out a copy of the MAR at src but with the signature block stripped.
*
* @param src The path of the source MAR file
* @param dest The path of the MAR file to write out that
has no signature block
* @return 0 on success
* -1 on error
*/
int
strip_signature_block(const char *src, const char * dest)
{
uint32_t offsetToIndex, dstOffsetToIndex, indexLength,
numSignatures = 0, leftOver;
int32_t stripAmount = 0;
int64_t oldPos, sizeOfEntireMAR = 0, realSizeOfSrcMAR, numBytesToCopy,
numChunks, i;
FILE *fpSrc = NULL, *fpDest = NULL;
int rv = -1, hasSignatureBlock;
char buf[BLOCKSIZE];
char *indexBuf = NULL;
if (!src || !dest) {
fprintf(stderr, "ERROR: Invalid parameter passed in.\n");
return -1;
}
fpSrc = fopen(src, "rb");
if (!fpSrc) {
fprintf(stderr, "ERROR: could not open source file: %s\n", src);
goto failure;
}
fpDest = fopen(dest, "wb");
if (!fpDest) {
fprintf(stderr, "ERROR: could not create target file: %s\n", dest);
goto failure;
}
/* Determine if the source MAR file has the new fields for signing or not */
if (get_mar_file_info(src, &hasSignatureBlock, NULL, NULL, NULL, NULL)) {
fprintf(stderr, "ERROR: could not determine if MAR is old or new.\n");
goto failure;
}
/* MAR ID */
if (ReadAndWrite(fpSrc, fpDest, buf, MAR_ID_SIZE, "MAR ID")) {
goto failure;
}
/* Offset to index */
if (fread(&offsetToIndex, sizeof(offsetToIndex), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could not read offset\n");
goto failure;
}
offsetToIndex = ntohl(offsetToIndex);
/* Get the real size of the MAR */
oldPos = ftello(fpSrc);
if (fseeko(fpSrc, 0, SEEK_END)) {
fprintf(stderr, "ERROR: Could not seek to end of file.\n");
goto failure;
}
realSizeOfSrcMAR = ftello(fpSrc);
if (fseeko(fpSrc, oldPos, SEEK_SET)) {
fprintf(stderr, "ERROR: Could not seek back to current location.\n");
goto failure;
}
if (hasSignatureBlock) {
/* Get the MAR length and adjust its size */
if (fread(&sizeOfEntireMAR,
sizeof(sizeOfEntireMAR), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could read mar size\n");
goto failure;
}
sizeOfEntireMAR = NETWORK_TO_HOST64(sizeOfEntireMAR);
if (sizeOfEntireMAR != realSizeOfSrcMAR) {
fprintf(stderr, "ERROR: Source MAR is not of the right size\n");
goto failure;
}
/* Get the num signatures in the source file so we know what to strip */
if (fread(&numSignatures, sizeof(numSignatures), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could read num signatures\n");
goto failure;
}
numSignatures = ntohl(numSignatures);
for (i = 0; i < numSignatures; i++) {
uint32_t signatureLen;
/* Skip past the signature algorithm ID */
if (fseeko(fpSrc, sizeof(uint32_t), SEEK_CUR)) {
fprintf(stderr, "ERROR: Could not skip past signature algorithm ID\n");
}
/* Read in the length of the signature so we know how far to skip */
if (fread(&signatureLen, sizeof(uint32_t), 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could not read signatures length.\n");
return CryptoX_Error;
}
signatureLen = ntohl(signatureLen);
/* Skip past the signature */
if (fseeko(fpSrc, signatureLen, SEEK_CUR)) {
fprintf(stderr, "ERROR: Could not skip past signature algorithm ID\n");
}
stripAmount += sizeof(uint32_t) + sizeof(uint32_t) + signatureLen;
}
} else {
sizeOfEntireMAR = realSizeOfSrcMAR;
numSignatures = 0;
}
if (((int64_t)offsetToIndex) > sizeOfEntireMAR) {
fprintf(stderr, "ERROR: Offset to index is larger than the file size.\n");
goto failure;
}
dstOffsetToIndex = offsetToIndex;
if (!hasSignatureBlock) {
dstOffsetToIndex += sizeof(sizeOfEntireMAR) + sizeof(numSignatures);
}
dstOffsetToIndex -= stripAmount;
/* Write out the index offset */
dstOffsetToIndex = htonl(dstOffsetToIndex);
if (fwrite(&dstOffsetToIndex, sizeof(dstOffsetToIndex), 1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write offset to index\n");
goto failure;
}
dstOffsetToIndex = ntohl(dstOffsetToIndex);
/* Write out the new MAR file size */
if (!hasSignatureBlock) {
sizeOfEntireMAR += sizeof(sizeOfEntireMAR) + sizeof(numSignatures);
}
sizeOfEntireMAR -= stripAmount;
/* Write out the MAR size */
sizeOfEntireMAR = HOST_TO_NETWORK64(sizeOfEntireMAR);
if (fwrite(&sizeOfEntireMAR, sizeof(sizeOfEntireMAR), 1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write size of MAR\n");
goto failure;
}
sizeOfEntireMAR = NETWORK_TO_HOST64(sizeOfEntireMAR);
/* Write out the number of signatures, which is 0 */
numSignatures = 0;
if (fwrite(&numSignatures, sizeof(numSignatures), 1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write out num signatures\n");
goto failure;
}
/* Write out the rest of the MAR excluding the index header and index
offsetToIndex unfortunately has to remain 32-bit because for backwards
compatibility with the old MAR file format. */
if (ftello(fpSrc) > ((int64_t)offsetToIndex)) {
fprintf(stderr, "ERROR: Index offset is too small.\n");
goto failure;
}
numBytesToCopy = ((int64_t)offsetToIndex) - ftello(fpSrc);
numChunks = numBytesToCopy / BLOCKSIZE;
leftOver = numBytesToCopy % BLOCKSIZE;
/* Read each file and write it to the MAR file */
for (i = 0; i < numChunks; ++i) {
if (ReadAndWrite(fpSrc, fpDest, buf, BLOCKSIZE, "content block")) {
goto failure;
}
}
/* Write out the left over */
if (ReadAndWrite(fpSrc, fpDest, buf,
leftOver, "left over content block")) {
goto failure;
}
/* Length of the index */
if (ReadAndWrite(fpSrc, fpDest, &indexLength,
sizeof(indexLength), "index length")) {
goto failure;
}
indexLength = ntohl(indexLength);
/* Consume the index and adjust each index by the difference */
indexBuf = malloc(indexLength);
if (fread(indexBuf, indexLength, 1, fpSrc) != 1) {
fprintf(stderr, "ERROR: Could not read index\n");
goto failure;
}
/* Adjust each entry in the index */
if (hasSignatureBlock) {
AdjustIndexContentOffsets(indexBuf, indexLength, -stripAmount);
} else {
AdjustIndexContentOffsets(indexBuf, indexLength,
sizeof(sizeOfEntireMAR) +
sizeof(numSignatures) -
stripAmount);
}
if (fwrite(indexBuf, indexLength, 1, fpDest) != 1) {
fprintf(stderr, "ERROR: Could not write index\n");
goto failure;
}
rv = 0;
failure:
if (fpSrc) {
fclose(fpSrc);
}
if (fpDest) {
fclose(fpDest);
}
if (rv) {
remove(dest);
}
if (indexBuf) {
free(indexBuf);
}
if (rv) {
remove(dest);
}
return rv;
}
void UI_MIT2EDFwindow::SelectFileButton()
{
FILE *header_inputfile=NULL,
*data_inputfile=NULL,
*annot_inputfile=NULL;
int i, j, p, len, hdl, *buf;
char header_filename[MAX_PATH_LENGTH],
txt_string[2048],
edf_filename[MAX_PATH_LENGTH],
data_filename[MAX_PATH_LENGTH],
annot_filename[MAX_PATH_LENGTH],
filename_x[MAX_PATH_LENGTH],
scratchpad[4096],
*charpntr;
unsigned char a_buf[128];
long long filesize;
pushButton1->setEnabled(false);
strcpy(header_filename, QFileDialog::getOpenFileName(0, "Select inputfile", QString::fromLocal8Bit(recent_opendir), "MIT header files (*.hea *.HEA)").toLocal8Bit().data());
if(!strcmp(header_filename, ""))
{
pushButton1->setEnabled(true);
return;
}
get_directory_from_path(recent_opendir, header_filename, MAX_PATH_LENGTH);
header_inputfile = fopeno(header_filename, "rb");
if(header_inputfile==NULL)
{
snprintf(txt_string, 2048, "Can not open file %s for reading.\n", header_filename);
textEdit1->append(QString::fromLocal8Bit(txt_string));
pushButton1->setEnabled(true);
return;
}
get_filename_from_path(filename_x, header_filename, MAX_PATH_LENGTH);
snprintf(txt_string, 2048, "Read file: %s", filename_x);
textEdit1->append(QString::fromLocal8Bit(txt_string));
remove_extension_from_filename(filename_x);
charpntr = fgets(scratchpad, 4095, header_inputfile);
if(charpntr == NULL)
{
textEdit1->append("Can not read header file. (error 1)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
len = strlen(charpntr);
if(len < 6)
{
textEdit1->append("Can not read header file. (error 2)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
for(i=0; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == len)
{
textEdit1->append("Can not read header file. (error 3)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
if(strcmp(charpntr, filename_x))
{
textEdit1->append("Can not read header file. (error 4)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
textEdit1->append("Can not read header file. (error 5)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
mit_hdr.chns = atoi(charpntr + p);
if(mit_hdr.chns < 1)
{
textEdit1->append("Error, number of signals is less than one. (error 6)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
if(mit_hdr.chns > MAXSIGNALS)
{
textEdit1->append("Error, Too many signals in header. (error 7)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
textEdit1->append("Can not read header file. (error 8)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
mit_hdr.sf = atoi(charpntr + p);
if(mit_hdr.sf < 1)
{
textEdit1->append("Error, samplefrequency is less than 1 Hz. (error 9)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
if(mit_hdr.sf > 100000)
{
textEdit1->append("Error, samplefrequency is more than 100000 Hz. (error 10)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
mit_hdr.smp_period = 1000000000LL / mit_hdr.sf;
strcat(filename_x, ".dat");
for(j=0; j<mit_hdr.chns; j++)
{
mit_hdr.adc_gain[j] = 200.0;
mit_hdr.adc_resolution[j] = 12;
mit_hdr.adc_zero[j] = 0;
mit_hdr.init_val[j] = 0;
sprintf(mit_hdr.label[j], "chan. %i", j + 1);
charpntr = fgets(scratchpad, 4095, header_inputfile);
if(charpntr == NULL)
{
textEdit1->append("Can not read header file. (error 11)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
len = strlen(charpntr);
if(len < 6)
{
textEdit1->append("Can not read header file. (error 12)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
for(i=0; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == len)
{
textEdit1->append("Can not read header file. (error 13)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
if(strcmp(charpntr, filename_x))
{
textEdit1->append("Error, filenames are different. (error 14)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == len)
{
textEdit1->append("Can not read header file. (error 15)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
mit_hdr.format[j] = atoi(charpntr + p);
if((mit_hdr.format[j] != 212) && (mit_hdr.format[j] != 16))
{
snprintf(txt_string, 2048, "Error, unsupported format: %i (error 16)\n", mit_hdr.format[j]);
textEdit1->append(txt_string);
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
if(j>0)
{
if(mit_hdr.format[j] != mit_hdr.format[0])
{
textEdit1->append("Error, different formats in the same file. (error 17)\n");
fclose(header_inputfile);
pushButton1->setEnabled(true);
return;
}
}
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
mit_hdr.adc_gain[j] = atoi(charpntr + p);
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
mit_hdr.adc_resolution[j] = atoi(charpntr + p);
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
mit_hdr.adc_zero[j] = atoi(charpntr + p);
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
mit_hdr.init_val[j] = atoi(charpntr + p);
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
// skip
p = ++i;
for(; i<len; i++)
{
if(charpntr[i] == ' ')
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
// skip
p = ++i;
for(; i<len; i++)
{
if((charpntr[i] == '\n') || (charpntr[i] == '\r'))
{
charpntr[i] = 0;
break;
}
}
if(i == p)
{
continue;
}
strncpy(mit_hdr.label[j], charpntr + p, 16);
mit_hdr.label[j][16] = 0;
}
fclose(header_inputfile);
strcpy(data_filename, header_filename);
remove_extension_from_filename(data_filename);
strcpy(edf_filename, data_filename);
strcpy(annot_filename, data_filename);
strcat(data_filename, ".dat");
strcat(edf_filename, ".edf");
strcat(annot_filename, ".atr");
data_inputfile = fopeno(data_filename, "rb");
if(data_inputfile==NULL)
{
snprintf(txt_string, 2048, "Can not open file %s for reading.\n", data_filename);
textEdit1->append(QString::fromLocal8Bit(txt_string));
pushButton1->setEnabled(true);
return;
}
fseeko(data_inputfile, 0LL, SEEK_END);
filesize = ftello(data_inputfile);
if(filesize < (mit_hdr.chns * mit_hdr.sf * 45 / 10))
{
textEdit1->append("Error, .dat filesize is too small.\n");
fclose(data_inputfile);
pushButton1->setEnabled(true);
return;
}
mit_hdr.sf_div = 1;
mit_hdr.sf_block = mit_hdr.sf;
if(!(mit_hdr.sf % 10))
{
mit_hdr.sf_div = 10;
mit_hdr.sf_block /= mit_hdr.sf_div;
}
else if(!(mit_hdr.sf % 8))
{
mit_hdr.sf_div = 8;
mit_hdr.sf_block /= mit_hdr.sf_div;
}
else if(!(mit_hdr.sf % 4))
{
mit_hdr.sf_div = 4;
mit_hdr.sf_block /= mit_hdr.sf_div;
}
else if(!(mit_hdr.sf % 2))
{
mit_hdr.sf_div = 2;
mit_hdr.sf_block /= mit_hdr.sf_div;
}
hdl = edfopen_file_writeonly(edf_filename, EDFLIB_FILETYPE_EDFPLUS, mit_hdr.chns);
if(hdl<0)
{
snprintf(txt_string, 2048, "Can not open file %s for writing.\n", edf_filename);
textEdit1->append(QString::fromLocal8Bit(txt_string));
fclose(data_inputfile);
pushButton1->setEnabled(true);
return;
}
for(i=0; i<mit_hdr.chns; i++)
{
if(edf_set_samplefrequency(hdl, i, mit_hdr.sf_block))
{
textEdit1->append("Error: edf_set_samplefrequency()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
}
for(i=0; i<mit_hdr.chns; i++)
{
if(edf_set_digital_minimum(hdl, i, -32768))
{
textEdit1->append("Error: edf_set_digital_minimum()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
}
for(i=0; i<mit_hdr.chns; i++)
{
if(edf_set_digital_maximum(hdl, i, 32767))
{
textEdit1->append("Error: edf_set_digital_maximum()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
}
for(i=0; i<mit_hdr.chns; i++)
{
if(edf_set_label(hdl, i, mit_hdr.label[i]))
{
textEdit1->append("Error: edf_set_label()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
}
for(i=0; i<mit_hdr.chns; i++)
{
if(edf_set_physical_dimension(hdl, i, "uV"))
{
textEdit1->append("Error: edf_set_physical_dimension()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
if(edf_set_physical_maximum(hdl, i, (double)((32767 - mit_hdr.adc_zero[i]) * 1000) / mit_hdr.adc_gain[i]))
{
textEdit1->append("Error: edf_set_physical_maximum()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
if(edf_set_physical_minimum(hdl, i, (double)((-32768 - mit_hdr.adc_zero[i]) * 1000) / mit_hdr.adc_gain[i]))
{
textEdit1->append("Error: edf_set_physical_minimum()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
}
if(edf_set_datarecord_duration(hdl, 100000 / mit_hdr.sf_div))
{
textEdit1->append("Error: edf_set_datarecord_duration()\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
buf = (int *)malloc(mit_hdr.sf_block * mit_hdr.chns * sizeof(int));
if(buf == NULL)
{
textEdit1->append("Malloc() error (buf)\n");
fclose(data_inputfile);
edfclose_file(hdl);
pushButton1->setEnabled(true);
return;
}
/////////////////// Start conversion //////////////////////////////////////////
int k, blocks, tmp1, tmp2;
fseeko(data_inputfile, 0LL, SEEK_SET);
blocks = filesize / (mit_hdr.sf_block * mit_hdr.chns);
QProgressDialog progress("Converting digitized signals ...", "Abort", 0, blocks);
progress.setWindowModality(Qt::WindowModal);
progress.setMinimumDuration(200);
if(mit_hdr.format[0] == 212)
{
blocks *= 10;
blocks /= 15;
progress.setMaximum(blocks);
for(k=0; k<blocks; k++)
{
if(!(k % 100))
{
progress.setValue(k);
qApp->processEvents();
if(progress.wasCanceled() == true)
{
textEdit1->append("Conversion aborted by user.\n");
fclose(data_inputfile);
edfclose_file(hdl);
free(buf);
pushButton1->setEnabled(true);
return;
}
}
for(i=0; i<mit_hdr.sf_block; i++)
{
for(j=0; j<mit_hdr.chns; j++)
{
if(j % 2)
{
tmp1 = fgetc(data_inputfile);
tmp2 = fgetc(data_inputfile);
if(tmp2 == EOF)
{
goto OUT;
}
buf[j * mit_hdr.sf_block + i] = (tmp1 & 0xf0) << 4;
buf[j * mit_hdr.sf_block + i] += tmp2;
if(buf[j * mit_hdr.sf_block + i] & 0x800)
{
buf[j * mit_hdr.sf_block + i] |= 0xfffff000;
}
}
else
{
tmp1 = fgetc(data_inputfile);
tmp2 = fgetc(data_inputfile);
buf[j * mit_hdr.sf_block + i] = (tmp2 & 0x0f) << 8;
buf[j * mit_hdr.sf_block + i] += tmp1;
if(buf[j * mit_hdr.sf_block + i] & 0x800)
{
buf[j * mit_hdr.sf_block + i] |= 0xfffff000;
}
fseeko(data_inputfile, -1LL, SEEK_CUR);
}
}
}
if(edf_blockwrite_digital_samples(hdl, buf))
{
progress.reset();
textEdit1->append("A write error occurred during conversion.\n");
fclose(data_inputfile);
edfclose_file(hdl);
free(buf);
pushButton1->setEnabled(true);
return;
}
}
}
if(mit_hdr.format[0] == 16)
{
blocks /= 2;
progress.setMaximum(blocks);
for(k=0; k<blocks; k++)
{
if(!(k % 100))
{
progress.setValue(k);
qApp->processEvents();
if(progress.wasCanceled() == true)
{
textEdit1->append("Conversion aborted by user.\n");
fclose(data_inputfile);
edfclose_file(hdl);
free(buf);
pushButton1->setEnabled(true);
return;
}
}
for(i=0; i<mit_hdr.sf_block; i++)
{
for(j=0; j<mit_hdr.chns; j++)
{
tmp1 = fgetc(data_inputfile);
if(tmp1 == EOF)
{
goto OUT;
}
tmp1 += (fgetc(data_inputfile) << 8);
if(tmp1 & 0x8000)
{
tmp1 |= 0xffff0000;
}
buf[j * mit_hdr.sf_block + i] = tmp1;
}
}
if(edf_blockwrite_digital_samples(hdl, buf))
{
progress.reset();
textEdit1->append("A write error occurred during conversion.\n");
fclose(data_inputfile);
edfclose_file(hdl);
free(buf);
pushButton1->setEnabled(true);
return;
}
}
}
OUT:
progress.reset();
qApp->processEvents();
/////////////////// End conversion //////////////////////////////////////////
fclose(data_inputfile);
free(buf);
int annot_code, tc=0, skip;
long long bytes_read;
get_filename_from_path(filename_x, annot_filename, MAX_PATH_LENGTH);
annot_inputfile = fopeno(annot_filename, "rb");
if(annot_inputfile==NULL)
{
remove_extension_from_filename(annot_filename);
strcat(annot_filename, ".ari");
annot_inputfile = fopeno(annot_filename, "rb");
}
if(annot_inputfile==NULL)
{
remove_extension_from_filename(annot_filename);
strcat(annot_filename, ".ecg");
annot_inputfile = fopeno(annot_filename, "rb");
}
if(annot_inputfile==NULL)
{
snprintf(txt_string, 2048, "Can not open file %s for reading.\n"
"Annotations can not be included.", filename_x);
textEdit1->append(QString::fromLocal8Bit(txt_string));
}
else
{
snprintf(txt_string, 2048, "Read file: %s", filename_x);
textEdit1->append(QString::fromLocal8Bit(txt_string));
fseeko(annot_inputfile, 0LL, SEEK_END);
filesize = ftello(annot_inputfile);
progress.setLabelText("Converting annotations ...");
progress.setMinimum(0);
progress.setMaximum(filesize);
fseeko(annot_inputfile, 0LL, SEEK_SET);
for(bytes_read=0LL; bytes_read < filesize; bytes_read += 2LL)
{
if(!(bytes_read % 100))
{
progress.setValue(bytes_read);
qApp->processEvents();
if(progress.wasCanceled() == true)
{
textEdit1->append("Conversion aborted by user.\n");
break;
}
}
skip = 0;
if(fread(a_buf, 2, 1, annot_inputfile) != 1)
{
break;
}
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
if(*((unsigned short *)a_buf) == 0) // end of file
{
break;
}
annot_code = a_buf[1] >> 2;
if(annot_code == 59)
{
if(fread(a_buf, 4, 1, annot_inputfile) != 1)
{
break;
}
tc += (*((unsigned short *)a_buf) << 16);
tc += *((unsigned short *)(a_buf + 2));
}
else if(annot_code == 63)
{
skip = *((unsigned short *)a_buf) & 0x3ff;
if(skip % 2) skip++;
}
else if((annot_code >= 0) && (annot_code <= ACMAX))
{
tc += *((unsigned short *)a_buf) & 0x3ff;
#pragma GCC diagnostic warning "-Wstrict-aliasing"
if(annot_code < 42)
{
edfwrite_annotation_latin1(hdl, ((long long)tc * mit_hdr.smp_period) / 100000LL, -1, annotdescrlist[annot_code]);
}
else
{
edfwrite_annotation_latin1(hdl, ((long long)tc * mit_hdr.smp_period) / 100000LL, -1, "user-defined");
}
}
if(skip)
{
if(fseek(annot_inputfile, skip, SEEK_CUR) < 0)
{
break;
}
bytes_read += skip;
}
}
fclose(annot_inputfile);
}
progress.reset();
edfclose_file(hdl);
textEdit1->append("Ready.\n");
pushButton1->setEnabled(true);
}
int main(int argc, char *argv[]) {
char buf[1024];
FILE *fa;
off_t start, stop;
off_t from, to;
off_t beginning;
int linewidth;
char c;
if (sizeof(off_t) < 8) {
fprintf(stderr, "WARNING: Not compiled with large file support\n");
}
if (argc != 4) {
fprintf(stderr, "Usage: %s fasta_file start stop\n", argv[0]);
return 1;
}
if ((fa = fopen(argv[1], "r")) == NULL) {
perror(argv[1]);
return 2;
}
start = atol(argv[2]);
stop = atol(argv[3]);
if (start > stop) {
from = start;
start = stop;
stop = from;
}
if (fgets(buf, 1023, fa) == NULL) {
perror(argv[1]);
return 2;
}
if (buf[0] != '>') {
fprintf(stderr, "%s: Not a fasta file\n", argv[1]);
return 1;
}
beginning = ftello(fa);
if (fgets(buf, 1023, fa) == NULL) {
perror(argv[1]);
return 2;
}
linewidth = strlen(buf) - 1;
if (buf[linewidth] != '\n') {
fprintf(stderr, "ERROR: FASTA sequence lines are too wide\n");
return 1;
}
from = beginning + start + ((start - 1)/ linewidth) - 1;
to = beginning + stop + ((stop - 1)/ linewidth) - 1;
if (fseeko(fa, to, SEEK_SET) < 0) {
perror("Seek error");
return 1;
}
fseeko(fa, from, SEEK_SET);
while (from++ <= to) {
c = fgetc(fa);
if (c != '\n') {
if (c == '>' || c == EOF) {
fprintf(stderr,
"WARNING: %d is past the end of the sequence\n", stop);
break;
}
putc(c, stdout);
}
}
fclose(fa);
putc('\n', stdout);
return 0;
}
