bool DiskPageCache::verify ( BigFile *f ) {
long vfd = f->getVfd();
// ensure validity
if ( vfd < 0 ) return true;
// this vfd may have already been nuked by call to unlink!
if ( ! m_memOff [ vfd ] ) return true;
// debug msg
//log("VERIFYING PAGECACHE vfd=%li fn=%s",vfd,f->getFilename());
// read into here
char buf [ 32 * 1024 ];//GB_PAGE_SIZE ]; //m_pageSize ];
// ensure threads disabled
bool on = ! g_threads.areThreadsDisabled();
if ( on ) g_threads.disableThreads();
// disable ourselves
disableCache();
// add valid offsets used by vfd into m_availMemOff
for ( long i = 0 ; i < m_maxPagesInFile [ vfd ] ; i++ ) {
long off = m_memOff [ vfd ] [ i ];
if ( off < 0 ) continue;
//char *p = getMemPtrFromOff ( off );
oldshort size = 0;
readFromCache(&size, off, OFF_SIZE, sizeof(oldshort));
//oldshort size = *(oldshort *)(p+OFF_SIZE);
oldshort skip = 0;
readFromCache(&skip, off, OFF_SKIP, sizeof(oldshort));
if ( size > 32 * 1024 ){
char *xx=NULL; *xx=0; }
//oldshort skip = *(oldshort *)(p+OFF_SKIP);
FileState fstate;
if ( ! f->read ( buf ,
size ,
((long long)i * (long long)m_pageSize) +
(long long)skip ,
&fstate ,
NULL , // state
NULL , // callback
0 )){// niceness
// core if it did not complete
char *xx = NULL; *xx = 0; }
// compare to what we have in mem
log("checking page # %li size=%li skip=%li", i, size, skip);
char buf2[32 * 1024];
readFromCache( buf2, off, HEADERSIZE + skip, size );
if ( memcmp ( buf, buf2, size ) != 0 ){
char *xx = NULL; *xx = 0; }
//if ( memcmp ( buf , p + HEADERSIZE + skip, size ) != 0 ) {
//char *xx = NULL; *xx = 0; }
}
if ( on ) g_threads.enableThreads();
enableCache();
// debug msg
log("DONE VERIFYING PAGECACHE");
return true;
}
// add data from "page" (we just read it from disk or wrote to disk)
// into "p" page in memory
void DiskPageCache::enhancePage (long poff, char *page, long size,
oldshort skip) {
oldshort psize = 0;
readFromCache( &psize, poff, OFF_SIZE, sizeof(oldshort));
//oldshort psize = *(oldshort *)(p+OFF_SIZE);
oldshort pskip = 0;
readFromCache( &pskip, poff, OFF_SKIP, sizeof(oldshort));
//oldshort pskip = *(oldshort *)(p+OFF_SKIP);
// can we add to front of page?
if ( skip < pskip ) {
long diff = pskip - skip;
// . we cored here because page[diff-1] was out of bounds. why?
// . do not allow gap in between cached data, that is, we have
// cached bytes at the end of the page, then we try to cache
// some at the beginning, and it's not contiguous... we are
// not built for that... this can happen when dumping a file,
// if your first reads up to the file end (somewhere in the
// middle of the page) and your second read starts somewhere
// else.... mmmm... i dunno....
if ( skip + size < pskip || diff > size ) {
log("db: Avoided cache gap in %s. diff=%li "
"size=%li pskip=%li skip=%li.",
m_dbname,diff,size,(long)pskip,(long)skip);
return;
}
writeToCache(poff, HEADERSIZE + skip , page , diff);
//memcpy ( p + HEADERSIZE + skip , page , diff );
psize += diff;
pskip -= diff;
writeToCache(poff, OFF_SIZE, &psize, sizeof(oldshort));
//*(oldshort *)(p+OFF_SIZE) = psize ;
writeToCache(poff, OFF_SKIP, &pskip, sizeof(oldshort));
//*(oldshort *)(p+OFF_SKIP) = pskip ;
}
// can we add to end of page?
long pend = pskip + psize;
long end = skip + size;
if ( end <= pend ) return;
long diff = end - pend ;
// if the read's starting point is beyond our ending point, bail,
// we don't want any holes...
if ( diff > size ) return;
writeToCache(poff, HEADERSIZE + pend, page + size - diff, diff);
//memcpy ( p + HEADERSIZE + pend , page + size - diff , diff );
oldshort tmp = psize+diff;
writeToCache(poff, OFF_SIZE, &tmp, sizeof(oldshort));
//*(oldshort *)(p+OFF_SIZE) = (oldshort)psize + diff;
}
// the link information is bytes 8-16 of each page in mem (next/prev mem ptrs)
void DiskPageCache::promotePage ( long poff , bool isNew ) {
if ( isNew ) {
here:
long tmp = -1;
writeToCache(poff, OFF_PREV, &tmp, sizeof(long));
// testing
readFromCache ( &tmp, poff, OFF_PREV, sizeof(long) );
if ( tmp != -1 ){
char *xx=NULL; *xx=0;}
//*(long *)(p + OFF_PREV) = -1 ;// our prev is -1 (none)
writeToCache(poff, OFF_NEXT, &m_headOff, sizeof(long));
//*(long *)(p+OFF_NEXT) = m_headOff;// our next is the old head
// the old head's prev is us
if ( m_headOff >= 0 ) {
writeToCache(m_headOff, OFF_PREV, &poff,
sizeof(long));
//char *headPtr = getMemPtrFromOff ( m_headOff ) ;
//*(long *)(headPtr + OFF_PREV) = poff;
}
// and we're the new head
m_headOff = poff;
// if no tail, we become that, too, we must be the first
if ( m_tailOff < 0 ) m_tailOff = poff;
return;
}
// otherwise, we have to excise
excisePage ( poff );
// and add as new
goto here;
}
// remove a page from the linked list
void DiskPageCache::excisePage ( long poff ) {
// get our neighbors, NULL if none
long prev = 0;
readFromCache(&prev, poff, OFF_PREV, sizeof(long));
//long prev = *(long *)(p + OFF_PREV);
long next = 0;
readFromCache(&next, poff, OFF_NEXT, sizeof(long));
//long next = *(long *)(p + OFF_NEXT);
// if we were the head or tail, then pass it off to our neighbor
if ( poff == m_headOff ) m_headOff = next;
if ( poff == m_tailOff ) m_tailOff = prev;
// our prev's next becomes our old next
if ( prev >= 0 ) {
//char *prevPtr = getMemPtrFromOff ( prev );
writeToCache(prev, OFF_NEXT, &next, sizeof(long));
//*(long *)(prevPtr + OFF_NEXT ) = next;
}
// our next's prev becomes our old prev
if ( next >= 0 ) {
//char *nextPtr = getMemPtrFromOff ( next );
writeToCache(next, OFF_PREV, &prev, sizeof(long));
//long *)(nextPtr + OFF_PREV ) = prev;
}
}
bool DictTool::init()
{
qDebug() << "Initializing dictionary...";
wordsDict.clear();
definitionsDict.clear();
if(QFile::exists("data/words.dat"))
{
if(!readFromCache()) return false;
}
else
{
if(!readFromFile()) return false;
}
qDebug() << "Done.";
qDebug() << "Words size:" << wordsDict.size();
qDebug() << "Definitions size:" << definitionsDict.size();
return true;
}
// skip is offset of "page" into physical page
void DiskPageCache::addPage(long vfd,long pageNum,char *page,long size,
oldshort skip){
// . if pageNum is beyond the file size
// . see the explanation for this same error msg above
if ( pageNum >= m_maxPagesInFile[vfd] ) {
// this has happened during a merge before!! (at startup)
//log(LOG_LOGIC,"db: pagecache: addPage: Bad engineer. "
// happens because rdbdump did not get a high enough
// maxfilesize so we did not make enough pages! we endedup
// dumping more than what was end the tree because stuff was
// added to the tree while dumping!
log("db: pagecache: Caught add breach. "
"pageNum=%li max=%li db=%s",
pageNum,m_maxPagesInFile[vfd],m_dbname);
return;
}
// debug msg
//log("addPage: pageNum=%li page[0]=%hhx size=%li skip=%li",
// pageNum,page[0],size,(long)skip);
long poff = m_memOff [ vfd ] [ pageNum ] ;
// p will be NULL if page does not have any data in memory yet
//char *p = getMemPtrFromOff ( poff );
// if page already exists in cache and needs data on the boundaries
// we may be able to supply it
if ( poff >= 0 ) {
// debug msg
//log("ENHANCING off=%li",poff);
enhancePage ( poff , page , size , skip );
return;
}
// don't add any more if we're minimizing disk seeks and are full
if ( m_minimizeDiskSeeks &&
m_numPagesPresentOfFile[vfd] >= m_maxPagesPerFile[vfd] )
return;
// top:
// try to get an available memory spot from list
if ( m_numAvailMemOffs > 0 ) {
poff = m_availMemOff [ --m_numAvailMemOffs ] ;
// debug msg
//log("RECYCLING off=%li",poff);
}
// can we grab a page from memory without having to grow?
else if ( m_nextMemOff + m_pageSize + HEADERSIZE < m_upperMemOff ) {
poff = m_nextMemOff;
m_nextMemOff += m_pageSize + HEADERSIZE;
// debug msg
//log("CLAIMING off=%li",poff);
}
// . we now grow everything at start
// . otherwise, try to grow the page cache by 200k
//else if ( m_nextMemOff + m_pageSize + HEADERSIZE < m_maxMemOff ) {
// // grow by 100k worth of pages each time
// if ( ! growCache ( m_upperMemOff + 200*1024 ) ) return;
// goto top;
//}
// this should never happen. Since in minimizeDiskSeek we have
// an exact number of pages per file
else if ( m_minimizeDiskSeeks ) {
char *xx = NULL; *xx = 0;
}
// if no freebies left, take over the tail page in memory
else {
poff = m_tailOff;
//char *p = getMemPtrFromOff ( poff );
excisePage ( poff );
// . the file no longer owns him
// . this is a long ptr to &m_bufOffs[vfd][pageNum]
// . if that vfd no longer exists it should have added all its
// pages to m_avail list
//long tmp = -1;
long *memOffPtr = NULL;
readFromCache(&memOffPtr, poff, OFF_PTR, sizeof(long*));
*memOffPtr = -1;
//m_cacheBuf.writeToCache(poff, OFF_PTR, &tmp, sizeof(long));
// testing
//m_cacheBuf.readFromCache ( &tmp, poff+OFF_PTR, sizeof(long) );
//if ( tmp != -1 ){
//char *xx=NULL; *xx=0;}
//**(long **)(p+OFF_PTR) = -1;
// debug msg
//log("KICKINGTAIL off=%li",poff);
}
// sanity check
if ( poff < 0 ) { char *xx = NULL; *xx = 0; }
// get ptr to the page in memory from the memory offset
//p = getMemPtrFromOff ( poff );
// store the size as first 2 bytes
writeToCache(poff, OFF_SIZE, &size, sizeof(oldshort));
// oldshort tmp = 0;
// m_cacheBuf.readFromCache ( &tmp, poff, OFF_SIZE, sizeof(long) );
// if ( tmp != size ){
// char *xx=NULL; *xx=0;}
//*(oldshort *)(p+OFF_SIZE) = size;
writeToCache( poff, OFF_SKIP, &skip, sizeof(oldshort) );
//*(oldshort *)(p+OFF_SKIP) = skip;
// sanity check
if ( size + skip > m_pageSize ) { char *xx = NULL; *xx = 0; }
// store the link information in bytes 8-16
promotePage ( poff , true/*isNew?*/ );
// then store a ptr to m_memOff[vfd][pageNum] so we can set *ptr
// to -1 if they page gets replaced by another
long *memOffPtr = &m_memOff[ vfd ][ pageNum ];
writeToCache( poff, OFF_PTR, &memOffPtr, sizeof(long*));
//*(long **)(p+OFF_PTR) = &m_memOff [ vfd ] [ pageNum ] ;
// then the data from disk (skip over link info)
writeToCache( poff, HEADERSIZE + skip, page, size);
//memcpy ( p + HEADERSIZE + skip , page , size );
// transform mem ptr to offset
if ( !m_useRAMDisk && ! m_useSHM ) {
long off = -1;
char *p = getMemPtrFromOff ( poff );
for ( long i = 0 ; i < m_numPageSets ; i++ ) {
if ( p < m_pageSet[i] ) continue;
if ( p > m_pageSet[i] + m_pageSetSize[i] )
continue;
off = p - m_pageSet[i] + i * m_maxPageSetSize ;
break;
}
// update map
m_memOff [ vfd ] [ pageNum ] = off;
// sanity check
if ( off != poff ) { char *xx=NULL; *xx=0; }
}
else
m_memOff [ vfd ] [ pageNum ] = poff;
// update the header of that page
// we have added the page!
if ( m_minimizeDiskSeeks )
m_numPagesPresentOfFile[vfd]++;
}
// . this returns true iff the entire read was copied into
// "buf" from the page cache
// . it will move the used pages to the head of the linked list
// . if *buf is NULL we allocate here
void DiskPageCache::getPages ( long vfd ,
char **buf ,
long numBytes ,
long long offset ,
long *newNumBytes ,
long long *newOffset ,
char **allocBuf ,
long *allocSize ,
long allocOff ) {
// check for override function
//if ( m_isOverriden ) {
// //log ( LOG_INFO, "cache: Get Pages [%li] [%li][%lli]",
// // vfd, numBytes, offset );
// m_getPages2 ( this,
// vfd,
// buf,
// numBytes,
// offset,
// newNumBytes,
// newOffset );
// return;
//}
// return new disk offset, assume unchanged
*newOffset = offset;
*newNumBytes = numBytes;
// return if no pages allowed in page cache
if ( m_maxMemOff == 0 ) return;
// or disabled
if ( ! m_enabled ) return;
// disabled at the master controls?
if ( m_switch && ! *m_switch ) return;
// or if minimizeDiskSeeks did not accept the vfd
if ( m_minimizeDiskSeeks && vfd < 0 )
return;
// or if no pages in this vfd
if ( !m_memOff[vfd] )
return;
// debug point
//if ( offset == 16386 && numBytes == 16386 )
// log("hey");
// what is the page range?
long sp = offset / m_pageSize ;
long ep = (offset + (numBytes-1)) / m_pageSize ;
// . sanity check
// . we establish the maxPagesInFile when BigFile::open is called
// by RdbDump. Rdb.cpp calls m_dump.set with a maxFileSize based on
// the mem occupied by the RdbTree. BUT, recs can be added to the tree
// WHILE we are dumping, so we end up with a bigger file, and this
// disk page cache is not prepared for it!
if ( ep >= m_maxPagesInFile[vfd] ) {
// happens because rdbdump did not get a high enough
// maxfilesize so we did not make enough pages! we endedup
// dumping more than what was end the tree because stuff was
// added to the tree while dumping!
log("db: pagecache: Caught get breach. "
"ep=%li max=%li vfd=%li", ep,m_maxPagesInFile[vfd] ,vfd);
return;
//char *xx = NULL; *xx = 0;
}
char *bufPtr = *buf;
char *bufEnd = *buf + numBytes;
// our offset into first page on disk
oldshort start1 = offset - sp * m_pageSize;
// this is for second while loop
oldshort start2 = 0;
if ( ep == sp ) start2 = start1;
// store start pages
while ( sp <= ep ) {
// the page offset in memory
long poff = m_memOff[vfd][sp];
// get a ptr to it
//char *s = getMemPtrFromOff ( poff );
// break if we do not have page in memory
//if ( ! s ) break;
if ( poff < 0 ) break;
// first 2 bytes of page is how many bytes are used in page
oldshort size = 0;
readFromCache( &size, poff, OFF_SIZE, sizeof(oldshort));
//oldshort size = *(oldshort *)(s+OFF_SIZE);
// second set of 2 bytes is offset of data from page boundary
oldshort skip = 0;
readFromCache( &skip, poff, OFF_SKIP, sizeof(oldshort));
//oldshort skip = *(oldshort *)(s+OFF_SKIP);
// debug msg
//log("getPage: pageNum=%li page[0]=%hhx size=%li skip=%li",
// sp,s[HEADERSIZE],(long)size,(long)skip);
// if this page data starts AFTER our offset, it is no good
if ( skip > start1 ) break;
// adjust size by our page offset, we won't necessarily be
// starting our read at "skip"
size -= (start1 - skip);
// if size is 0 or less all cached data was below our offset
if ( size <= 0 ) break;
// . promote this page in the linked list
// . bytes 8-16 of each page in memory houses the
// next and prev ptrs to pages in memory
promotePage ( poff , false );
// allocate the read buffer if we need to
if ( ! *buf ) {
// allocate enough room for allocOff, too
long need = numBytes + allocOff;
char *p = (char *) mmalloc ( need,"PageCacheReadBuf" );
// let FileState know what needs to be freed
*allocBuf = p;
*allocSize = need;
// if couldn't allocate, return now, what's the point
if ( ! p ) return;
// let caller know his new read buffer
*buf = p + allocOff;
// assign the ptrs now
bufPtr = *buf ;
bufEnd = *buf + numBytes;
}
// don't store more than asked for
if ( bufPtr + size > bufEnd ) size = bufEnd - bufPtr;
readFromCache(bufPtr, poff, HEADERSIZE + start1 , size);
//memcpy ( bufPtr , s + HEADERSIZE + start1 , size );
bufPtr += size;
*newOffset += size;
*newNumBytes -= size;
// return if we got it all
if ( bufPtr >= bufEnd ) { m_hits += 1; return; }
// otherwise, advance to next page
sp++;
// and our page relative offset is zero now, iff ep > sp
if ( sp <= ep ) start1 = 0;
// if the cached page ended before the physical page, break out
// because we don't want any holes
readFromCache( &size, poff, OFF_SIZE, sizeof(oldshort));
if ( skip + size < m_pageSize ) break;
//if ( skip + *(oldshort *)(s+OFF_SIZE) < m_pageSize ) break;
}
// now store from tail down
/*
while ( ep > sp ) {
// the page offset in memory
long poff = m_memOff[vfd][ep];
// get a ptr to it
char *s = getMemPtrFromOff ( poff );
// break if we do not have page in memory
if ( ! s ) break;
// first 2 bytes of page is how many bytes are used
oldshort size = *(oldshort *)s;
// second set of 2 bytes is offset from boundary
oldshort skip = *(oldshort *)(s+OFF_SKIP);
// adjust size by our page offset, if not zero
if ( start2 > skip ) size -= (start2 - skip);
// his skip point could be beyond us, too
if ( skip >
// . promote this page in the linked list
// . bytes 8-16 of each page in memory houses the
// next and prev ptrs to pages in memory
promotePage ( s , poff , false );
// don't store more than asked for
if ( bufEnd - size < bufPtr ) size = bufEnd - bufPtr;
memcpy ( bufEnd - size , s + HEADERSIZE + start2 , size );
bufEnd -= size;
*newNumBytes -= size;
// return if we got it all
if ( bufEnd <= bufPtr ) { m_hits += 1; return; }
// if this page had a skip, break out, we don't wany any holes
if ( skip > 0 ) break;
// otherwise, advance to next page
ep--;
}
*/
m_misses += 1;
}
int main(int argc, char *argv[]){
cache_params_t params;
/* file handling stuff */
FILE *tracefile;
char instruction_ptr[50];
char instruction;
char mem_addr[50];
int count;
char *traceData;
Cache_t cache;
int tagSize, setSize, offsetSize;
if(argc == 2 && strcmp(argv[1], "-h") == 0){
printf("help menu\n");
return 1;
}
if(argc < 2 || argc > 6){
printf("ERROR: incorrect number of arguments\n");
return 1;
}
/*
* TODO:
* Check cachesize = numsets x setsize x blocksize
*/
/* Check if cachesize here and correct*/
/* should check that these are numbers as well */
if(isPowerOfTwo(atoi(argv[1]))){
params.cachesize = atoi(argv[1]);
}else{
printf("ERROR: cachesize must be a power of 2\n");
return 1;
}
/* check if blocksize is here and correct */
if(isPowerOfTwo(atoi(argv[3]))){
params.blocksize = atoi(argv[3]);
}else{
printf("ERROR: blocksize must be a power of 2\n");
return 1;
}
params.setsize = 1;
/* check for associativit y */
if(strcmp("direct", argv[2]) == 0){
params.associativity = "direct";
}else if( strcmp("assoc", argv[2]) == 0){
params.associativity = "assoc";
}else{
int i = 0, digit = 0;
char* str = argv[2];
char* test;
while(str[i] != '\0'){
if(isdigit(str[i])){
digit = i;
break;
}
i++;
}
/*
* setsize = 1 for d-m caches
*/
params.setsize = 1;
test = malloc(strlen(argv[2]));
strncpy(test, argv[2], digit);
if(strcmp("assoc:", test) == 0){
params.associativity = "n-way";
if(isPowerOfTwo( argv[2][digit] - '0')){
params.setsize = argv[2][digit] - '0';
}else{
printf("ERROR: n must be power of 2\n");
return 1;
}
}else{
printf("ERROR: invalid associativity format\n");
return 1;
}
}
/* check for writepolicy*/
if(strcmp("wt", argv[4]) == 0){
params.writepolicy = "wt";
}else if(strcmp("wb", argv[4]) == 0){
params.writepolicy = "wb";
}else{
printf("ERROR: invalid writepolicy format\n");
return 1;
}
/* check for tracefile */
if(!(tracefile = fopen(argv[5], "r"))){
printf("ERROR: could not find tracefile: %s\n", argv[5]);
return 1;
}else{
traceData = readFile(argv[5]);
}
/*
<<<<<<< HEAD
* create the new cache, do initialization stuff
=======
* create the new cache, do initializatino stuff
>>>>>>> 59f4c294c92c026c0d9d15ccfd2984234f2a3158
*/
cache = *newCache(params.cachesize, params.blocksize, params.setsize, params.associativity, params.writepolicy);
/*
* read each line of the tracefile
*
* DIRECT MAP ONLY RIGHT NOW
*/
count = 0;
while(fscanf(tracefile, "%s %c %s", instruction_ptr, &instruction, mem_addr) == 3) {
int tag_size;
int lines;
int offset_size;
int set_size;
int set_lines;
unsigned int addr_int;
char *addr_bin;
char *formatted_bin;
char *tag;
char *setid;
char *offset;
formatted_bin = (char*) malloc(sizeof(char) * 35);
addr_bin = (char*) malloc(sizeof(char) * 33);
lines = (int)(params.cachesize / params.blocksize);
set_lines = (int)(params.cachesize / (params.blocksize * params.setsize));
if(strcmp(params.associativity,"n-way") == 0){
set_size = (int) ceil((log10(set_lines)/log10(2.0)));
}else{
set_size = (int) ceil((log10(lines)/log10(2.0)));
}
offset_size = ceil((log10(params.blocksize)/log10(2.0)));
tag_size = (32 - (offset_size+set_size));
/*
* just for output
*/
tagSize = tag_size;
setSize = set_size;
offsetSize = offset_size;
tag = (char*) malloc(sizeof(char)*tag_size+1);
setid = (char*) malloc(sizeof(char)*set_size+1);
offset = (char*) malloc(sizeof(char)*offset_size+1);
addr_int = htoi(mem_addr);
addr_bin = itob(addr_int);
memcpy(tag, &addr_bin[0], tag_size);
tag[tag_size+1] = '\0';
memcpy(setid, &addr_bin[tag_size], set_size);
setid[set_size+1] = '\0';
memcpy(offset, &addr_bin[(tag_size+set_size)], offset_size);
offset[offset_size+1] = '\0';
if(DEBUG) {
printf("------\n%s\n",mem_addr);
printf("%s %s %s \n", tag, setid, offset);
printf("tag: %i, set: %i, offset: %i\n", tagSize, setSize, offsetSize);
}
/*
* Write to the cache
*/
if('W' == instruction){
if(DEBUG){
printf("Write instruction\n Cache State:\n");
printCache(&cache);
if( isInCache(&cache, addr_bin, tag, setid, offset)){
printf("\ncache already contains %s\n",mem_addr);
}else{
printf("\ncache does not contain %s\n",mem_addr);
}
if( isInCache(&cache, addr_bin, tag, setid, offset)){
printf("\ncache contains %s\n",mem_addr);
}
printf("\nCache State:\n");
printCache(&cache);
}
writeToCache(&cache, addr_bin, tag, setid, offset, lines);
/*
* Read from the cache
*/
}else if('R' == instruction ){
if(DEBUG) {
printf("Read instruction\n Cache State:\n");
printCache(&cache);
if( isInCache(&cache, addr_bin, tag, setid, offset)){
printf("\ncache already contains %s\n",mem_addr);
}else{
printf("\ncache does not contain %s\n",mem_addr);
}
if( isInCache(&cache, addr_bin, tag, setid, offset)){
printf("\ncache contains %s\n",mem_addr);
}
printf("\nCache State:\n");
printCache(&cache);
}
readFromCache(&cache, addr_bin, tag, setid, offset, lines);
}
count++;
}
printCache(&cache);
return 1;
}
void outputBlocks(struct lineFile *lf,
struct block *blockList, int score, FILE *f, boolean isRc,
char *qName, int qSize, char *qNibDir, struct dlList *qCache,
char *tName, int tSize, char *tNibDir, struct dlList *tCache,
boolean rescore)
/* Output block list as an axt to file f. */
{
int qStart = BIGNUM, qEnd = 0, tStart = BIGNUM, tEnd = 0;
struct block *lastBlock = NULL;
struct block *block;
struct dyString *qSym = newDyString(16*1024);
struct dyString *tSym = newDyString(16*1024);
struct dnaSeq *qSeq = NULL, *tSeq = NULL, *seq = NULL;
struct axt axt;
boolean qIsTwoBit = twoBitIsFile(qNibDir);
boolean tIsTwoBit = twoBitIsFile(tNibDir);
if (blockList == NULL)
return;
/* Figure overall dimensions. */
for (block = blockList; block != NULL; block = block->next)
{
if (qStart > block->qStart) qStart = block->qStart;
if (qEnd < block->qEnd) qEnd = block->qEnd;
if (tStart > block->tStart) tStart = block->tStart;
if (tEnd < block->tEnd) tEnd = block->tEnd;
}
/* Load sequence covering alignment from nib files. */
if (isRc)
{
reverseIntRange(&qStart, &qEnd, qSize);
if (qIsFa)
{
for (seq = qFaList ; seq != NULL ; seq = seq->next)
if (sameString(qName, seq->name))
break;
if (seq != NULL)
{
AllocVar(qSeq);
qSeq->size = qEnd - qStart;
qSeq->name = cloneString(qName);
qSeq->dna = cloneMem((seq->dna)+qStart, qSeq->size);
}
else
errAbort("sequence not found %s\n",qName);
}
else
qSeq = readFromCache(qCache, qNibDir, qName, qStart, qEnd - qStart, qSize, qIsTwoBit);
reverseIntRange(&qStart, &qEnd, qSize);
reverseComplement(qSeq->dna, qSeq->size);
}
else
{
if (qIsFa)
{
for (seq = qFaList ; seq != NULL ; seq = seq->next)
{
if (sameString(qName, seq->name))
break;
}
if (seq != NULL)
{
AllocVar(qSeq);
qSeq->size = qEnd - qStart;
qSeq->name = cloneString(qName);
qSeq->dna = (seq->dna)+qStart;
}
else
errAbort("sequence not found %s\n",qName);
}
else
qSeq = readFromCache(qCache, qNibDir, qName, qStart, qEnd - qStart, qSize, qIsTwoBit);
}
if (tIsFa)
{
for (seq = tFaList ; seq != NULL ; seq = seq->next)
if (sameString(tName, seq->name))
break;
if (seq != NULL)
{
AllocVar(tSeq);
tSeq->size = tEnd - tStart;
tSeq->name = cloneString(tName);
tSeq->dna = cloneMem((seq->dna)+tStart, tSeq->size);
}
else
errAbort("sequence not found %s\n",tName);
}
else
tSeq = readFromCache(tCache, tNibDir, tName, tStart, tEnd - tStart, tSize, tIsTwoBit);
/* Loop through blocks copying sequence into dynamic strings. */
for (block = blockList; block != NULL; block = block->next)
{
if (lastBlock != NULL)
{
int qGap = block->qStart - lastBlock->qEnd;
int tGap = block->tStart - lastBlock->tEnd;
if (qGap != 0 && tGap != 0)
{
errAbort("Gaps in both strand on alignment ending line %d of %s",
lf->lineIx, lf->fileName);
}
if (qGap > 0)
{
dyStringAppendMultiC(tSym, '-', qGap);
dyStringAppendN(qSym, qSeq->dna + lastBlock->qEnd - qStart, qGap);
}
if (tGap > 0)
{
dyStringAppendMultiC(qSym, '-', tGap);
dyStringAppendN(tSym, tSeq->dna + lastBlock->tEnd - tStart, tGap);
}
}
if (qSeq->size < block->qStart - qStart)
{
errAbort("read past end of sequence %s size =%d block->qStart-qstart=%d block->qStart=%d qEnd=%d \n", qName, qSeq->size, block->qStart-qStart,block->qStart, block->qEnd );
}
dyStringAppendN(qSym, qSeq->dna + block->qStart - qStart,
block->qEnd - block->qStart);
if (tSeq->size < block->tStart - tStart)
{
errAbort("read past end of sequence %s size =%d block->tStart-tstart=%d\n", tName, tSeq->size, block->tStart-tStart);
}
dyStringAppendN(tSym, tSeq->dna + block->tStart - tStart,
block->tEnd - block->tStart);
lastBlock = block;
}
if (qSym->stringSize != tSym->stringSize)
errAbort("qSize and tSize don't agree in alignment ending line %d of %s",
lf->lineIx, lf->fileName);
if (rescore)
score = axtScoreSym(scoreScheme, qSym->stringSize,
qSym->string, tSym->string);
/* Fill in an axt and write it to output. */
ZeroVar(&axt);
axt.qName = qName;
axt.qStart = qStart;
axt.qEnd = qEnd;
axt.qStrand = (isRc ? '-' : '+');
axt.tName = tName;
axt.tStart = tStart;
axt.tEnd = tEnd;
axt.tStrand = '+';
axt.score = score;
axt.symCount = qSym->stringSize;
axt.qSym = qSym->string;
axt.tSym = tSym->string;
axtWrite(&axt, f);
/* Clean up. */
if (!qIsFa)
freeDnaSeq(&qSeq);
freeDnaSeq(&tSeq);
dyStringFree(&qSym);
dyStringFree(&tSym);
}
