/** * Rebalance the segment list. The segment list is compacted and all * free segments are removed. The most frequent segment is placed at * the head of the list. * * Note: outSynapses must be updated after a call to this. */ void rebalanceSegments() { //const std::vector<UInt> &non_empties = getNonEmptySegList(); UInt bestOne = getMostActiveSegment(); // Swap the best one with the 0'th one if (bestOne != 0) { Segment seg = _segments[0]; _segments[0] = _segments[bestOne]; _segments[bestOne] = seg; } // Sort segments according to activation frequency // Redo free segments list _freeSegments.clear(); for (UInt segIdx = 0; segIdx != _segments.size(); ++segIdx) { if ( _segments[segIdx].empty() ) releaseSegment(segIdx); } }
const int FreeMgr::init(CommBlock &cb, DataBlock* blockZero, int freemgrType, IdxTreeGroupType chainType, int startBlock, int numberBlocks) { /** * Use this to allocate a group of blocks or sub blocks to a free list * startBlock is fbo NOT lbid * This allows us to use a simple for loop to traverese the full range of blocks * **/ int blkIdx, sbIdx; int rc; IdxEmptyListEntry emptyEntry; // populate the chains with pointer to empty entries FILE* indexFile; indexFile = cb.file.pFile; nullPtr(&emptyEntry); emptyEntry.type = EMPTY_PTR; if( isDebug( DEBUG_3 )) { printf("\nNew style Init v2, file: %u startBlock: %i num: %i \n", cb.file.oid, (unsigned int)startBlock, (unsigned int)numberBlocks); } int lastBlock = numberBlocks + startBlock; if (chainType == ENTRY_BLK) { emptyEntry.group = ENTRY_BLK; for ( blkIdx = lastBlock ; blkIdx > startBlock ; blkIdx-- ) { // if( isDebug( DEBUG_1 )) { printf("Initing block %u\n", blkIdx-1);} emptyEntry.fbo = mapLBID( cb, blkIdx-1, rc );// map fbo to lbid before storage if (rc != NO_ERROR ){ printf("DBG: Init 1: Error resolving LBID for OID: %u FBO: %u\n", cb.file.oid, blkIdx-1 ); return rc; } if( isDebug( DEBUG_3 )) { cout<<"DBG: Working on block "<<emptyEntry.fbo<<"\n"; } emptyEntry.sbid = 0; emptyEntry.type = EMPTY_PTR; rc = releaseSegment( cb, blockZero, freemgrType, chainType, &emptyEntry ); if (rc != NO_ERROR){ printf("Error releasing sb\n"); return rc; } } } else if (chainType == ENTRY_32 ){ emptyEntry.group = ENTRY_32; for ( blkIdx = lastBlock ; blkIdx > startBlock ; blkIdx-- ) { // if( isDebug( DEBUG_1 )) { printf("Initing block %u\n", blkIdx-1);} emptyEntry.fbo = mapLBID( cb, blkIdx-1, rc );// map fbo to lbid before storage if (rc != NO_ERROR ){ printf("DBG: Init 2: Error resolving LBID for OID: %u FBO: %u\n", cb.file.oid, blkIdx-1 ); return rc; } if( isDebug( DEBUG_3 )) { cout<<"DBG: Working on block "<<emptyEntry.fbo<<"\n"; } for (sbIdx=BYTE_PER_BLOCK/BYTE_PER_SUBBLOCK-1; sbIdx >-1; sbIdx--) { if( isDebug( DEBUG_3 )) { printf("DBG: Working on subblock %u\n", sbIdx); } emptyEntry.sbid = sbIdx; emptyEntry.type = EMPTY_PTR; rc = releaseSubblock( cb, blockZero, freemgrType, &emptyEntry ); if (rc != NO_ERROR){ printf("Error releasing sb\n"); return rc; } } } } else { if ( isDebug( DEBUG_1 )){ printf("DBG: Invalid segment size %i (should be ENTRY_32 or ENTRY_BLK)\n", chainType); } return ERR_INVALID_PARAM; } // now write sb0 back to disk uint64_t lbid = mapLBID( cb, 0, rc); if (rc != NO_ERROR ){ return rc; } rc = writeDBFile( cb, blockZero, lbid ); if (rc != NO_ERROR) { return rc; } return NO_ERROR; }
const int FreeMgr::assignSegment( CommBlock &cb, DataBlock* blockZero, const int freemgr_type, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr ) { int rc; // return code from file ops DataBlock workBlock; int listOffset; // entry in block zero of head pointer IdxEmptyListEntry emptyEntry; IdxEmptyListEntry emptyPtr; IdxEmptyListEntry emptyMap; uint64_t numBlocks; IdxEmptyListEntry newSb; FILE* indexFile; indexFile = cb.file.pFile; if( isDebug( DEBUG_3 )) { printf("DBG: Assign ENTRY_%i segment in %s\n", 1<<(unsigned int)segmentType,(freemgr_type==LIST)?"LIST":"TREE"); } /* Check all input parameters are ok */ if (!blockZero){ if( isDebug( DEBUG_1 )) { printf ("DBG: Bad pointer: blockZero is zero\n"); } return ERR_INVALID_PARAM; } if (!assignPtr) { if( isDebug( DEBUG_1 )) { printf ("DBG: Bad pointer: assignPtr is zero\n"); } return ERR_INVALID_PARAM; } if (freemgr_type != TREE && freemgr_type!= LIST) { if( isDebug( DEBUG_0 )) { printf ("DBG: assignSegment: Must be TREE or LIST\n");} return ERR_INVALID_PARAM; } numBlocks = getFileSize( indexFile )/BYTE_PER_BLOCK ; //find the start of the chain if (segmentType == ENTRY_32) { rc = assignSubblock( cb, blockZero, freemgr_type, assignPtr); if (rc != NO_ERROR) { if( isDebug( DEBUG_1 )) { printf("DBG: Error assigning sb (rc=%i)\n", rc); } } return rc; } listOffset = calcPtrOffset( segmentType ); if (freemgr_type==LIST && !(segmentType== ENTRY_4 || segmentType== ENTRY_BLK)) { if( isDebug( DEBUG_1 )) { printf("DBG: Assign segment size illegal %i\n", (unsigned int)segmentType); } return ERR_INVALID_PARAM; // should not have got here so quit with error } // read Empty Map in sb0 getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); if( isDebug( DEBUG_2 )) { cout<<"DBG: Empty map ENTRY_"<<(1<<segmentType)<<" was fbo "<<emptyPtr.fbo<<" sbid "<<(unsigned int)emptyPtr.sbid<<" entry "<<(unsigned int)emptyPtr.entry<<"\n"; } /* if (emptyPtr.fbo > numBlocks) { printf("DBG: Weirdness in assignSegment.. emptyPtr.fbo > numBlocks\n"); return ERR_FM_BAD_FBO; }*/ // check to see if queue has been built // if not then assign a container block, add LLP in entry 0 pointing to nothing if (emptyPtr.fbo == 0){ if( isDebug( DEBUG_3 )) { printf("DBG: Need to add sb to chain and entries to sb\n"); } // cannot assign more space to block list from a smaller list.. need to extend the file if ( segmentType == ENTRY_BLK ) { if ( isDebug( DEBUG_1 )){ printf("Out of space in BLOCK list, quitting\n"); } rc = extendFreespace( cb, blockZero, freemgr_type ); if (rc != NO_ERROR){ if (isDebug( DEBUG_3 )){ printf("DBG: Error extending file\n"); } return rc; } } rc = assignSubblock( cb, blockZero, freemgr_type, &emptyPtr); if (rc != NO_ERROR){ printf("DBG: Error extending chain\n"); return rc; } rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); if (rc != NO_ERROR){ /*printf("DBG: Error reading newly allocated sb\n");*/ return rc; } // update map to point to new bucket setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr); nullPtr(&emptyEntry); emptyEntry.type = EMPTY_LIST; emptyEntry.group = segmentType; setSubBlockEntry( workBlock.data, emptyPtr.sbid, 0, 8, &emptyEntry); // store head ptr rc = writeDBFile( cb, workBlock.data, emptyPtr.fbo ); if (rc != NO_ERROR){ return rc; } if( isDebug( DEBUG_2 )) { cout<<"DBG: Added fbo "<<emptyPtr.fbo<<" sbid "<<(unsigned int)emptyPtr.sbid<<" as bucket to chain ENTRY_"<<(1<<segmentType)<<"in"<<((freemgr_type==LIST)?"LIST":"TREE")<<"\n"; } } // follow the chain to the head container rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); if (rc != NO_ERROR) { if( isDebug( DEBUG_1 )) { cout<<"DBG: Error reading block ("<<emptyPtr.fbo<<") during segmentAssign: rc is "<<rc; } return rc; } getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); if ((emptyEntry.type != EMPTY_LIST) && (emptyEntry.type != EMPTY_PTR)) { if( isDebug( DEBUG_0 )) { printf("WTF: Bad entry in ENTRY_%i chain - type is %i (expected %i or %i)\n", 1<<segmentType, (unsigned int)emptyEntry.type, EMPTY_PTR, EMPTY_LIST ); printMemSubBlock( &workBlock, emptyPtr.sbid ); } if( isDebug( DEBUG_2 )) { cout<<"DBG: fbo "<<emptyEntry.fbo<<" sbid "<<(unsigned int)emptyEntry.sbid<<" entry "<< (unsigned int)emptyEntry.entry<<" chain ENTRY_"<<(1<<segmentType)<<"in"<<((freemgr_type==LIST)?"LIST":"TREE")<<"\n"; } return ERR_FM_BAD_TYPE; } if ((emptyEntry.fbo == 0) && (emptyEntry.type == EMPTY_PTR)) { if ( isDebug( DEBUG_0 )) {printf("DBG: Bad entry in %i list - found EMPTY_PTR but indicates block 0\n", 1<<segmentType );} return ERR_FM_BAD_TYPE; } if ((emptyEntry.fbo == 0) && (emptyEntry.type == EMPTY_LIST)) { /** * if at the end of the rainbow, in a bucket with no entries, fill the bucket * Allocate a sub block and split it into parts **/ // cannot assign more space to block list from a smaller list.. need to extend the file if ( segmentType == ENTRY_BLK ) { if ( isDebug( DEBUG_1 )){ printf("\nNeed to extend block\n");} if ( isDebug( DEBUG_1 )){ printf("Out of space in BLOCK list\n"); } rc = extendFreespace( cb, blockZero, freemgr_type ); if (rc != NO_ERROR){ if (isDebug( DEBUG_3 )){ printf("DBG: Error extending file\n"); } return rc; } getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); } else { rc = assignSubblock( cb, blockZero, freemgr_type, &newSb); if (rc != NO_ERROR){ // printf("DBG: Error extending chain\n"); return rc; } if (newSb.entry != 0){ printf("WTF: Entry should be 0 after assign from sb list, instead is %i", (unsigned int)newSb.entry); return ERR_FM_ASSIGN_ERR; } if (isDebug(DEBUG_2)){ cout<<"DBG: added fbo "<<newSb.fbo<<" sbid "<<(unsigned int)newSb.sbid<<" entry "<<(unsigned) newSb.entry<<" to "<<segmentType<<" list - need to split "<<(1<<(ENTRY_32-segmentType))<<"times\n"; } newSb.entry = 0; newSb.group = segmentType; newSb.type = EMPTY_PTR; emptyEntry = newSb; int idx, inc; inc = 1<<segmentType; for (idx=0; idx < ENTRY_PER_SUBBLOCK - inc; idx+= inc){ if( isDebug( DEBUG_3 )) { printf ("DBG: split..%i-%i\n", idx, idx+inc-1 );} newSb.entry = idx; releaseSegment( cb, blockZero, freemgr_type, segmentType, &newSb ); } emptyEntry.entry = idx; if( isDebug( DEBUG_3 )) { printf ("DBG: split and return..%i-%i\n", idx, idx+inc-1); } if( isDebug( DEBUG_2 )) { cout<<"DBG: Assigned fbo "<< emptyEntry.fbo<<" sbid "<<(unsigned int)emptyEntry.sbid<<" entry "<<(unsigned int)emptyEntry.entry<<" to chain ENTRY_"<<(1<<segmentType)<<"\n"; } memcpy(assignPtr, &emptyEntry, 8); uint64_t count; getSubBlockEntry( blockZero, 0, calcStatOffset(segmentType) , 8, &count ); count--; setSubBlockEntry( blockZero, 0, calcStatOffset(segmentType) , 8, &count ); return NO_ERROR; } } /** * got here because we didn't need to populate a chain and did not fall into any traps * so either we are at the end of bucket or we have a valid entry **/ if (emptyEntry.type == EMPTY_LIST) // reached end of this segment (should release it for re-use) { /** * release bucket **/ if( isDebug( DEBUG_2 )) { cout<<"DBG: Need to release sb fbo "<<emptyPtr.fbo<<" sbid "<<emptyPtr.sbid<<" from chain ENTRY_"<<(1<<segmentType)<<" in "<<((freemgr_type==LIST)?"LIST":"TREE")<<"\n"; } // when we stored the ptr in the empty map, we tweaked group, must change it back emptyPtr.type = EMPTY_PTR; emptyPtr.group = ENTRY_32; rc = releaseSubblock( cb, blockZero, freemgr_type, &emptyPtr ); if (rc != NO_ERROR) { printf("Error releasing sb\n"); return rc; } emptyPtr = emptyEntry; rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); if (rc != NO_ERROR){ printf("DBG: Error following chain\n"); return rc; } getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); } if (emptyEntry.type == EMPTY_PTR) { emptyPtr.entry--; blockZero->dirty = 1; setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); if( isDebug( DEBUG_3 )) { printf("DBG: Empty entry is now %u\n",(unsigned int)emptyPtr.entry); } if( isDebug( DEBUG_2 )) { cout<<"DBG: Assigned fbo "<<emptyEntry.fbo<<" sbid "<<emptyEntry.sbid<<" entry "<<emptyEntry.entry<<" from chain ENTRY_"<<(1<<segmentType)<<"\n"; cout<<"DBG: Empty map ENTRY_"<<(1<<segmentType)<<" now fbo "<<emptyPtr.fbo<<" sbid "<<emptyPtr.sbid<<" entry "<<emptyPtr.entry<<"\n"; } memcpy(assignPtr, &emptyEntry, 8); // -- workblock may have changed on disk since it was read.. making the writeDBfile dangerous without a readDBfile first nullPtr(&emptyMap); // zero out the entry readDBFile( cb, &workBlock, emptyPtr.fbo ); setSubBlockEntry( workBlock.data, emptyPtr.sbid, emptyPtr.entry+1, 8, &emptyMap ); rc = writeDBFile( cb, &workBlock, emptyPtr.fbo ); // -- uint64_t count; getSubBlockEntry( blockZero, 0, calcStatOffset(segmentType) , 8, &count ); count--; setSubBlockEntry( blockZero, 0, calcStatOffset(segmentType) , 8, &count ); return NO_ERROR; } return ERR_FM_ASSIGN_ERR; }
const int IndexList::deleteInBlock(const RID& rowId) { int width =LIST_ENTRY_WIDTH; int rc =ERR_IDX_LIST_INVALID_DELETE; IdxRidListPtr* lastIdxRidListPtr; IdxRidListPtr lastSubIdxRidListPtr; bool found; int type, count; IdxRidListPtr prevIdxRidListPtr; int prevSbid, prevEntry, prevType; uint64_t prevLbid; DataBlock prevDataBlock; int pos =0, totalbytes=0; int preTotalBytes, prevPos ; //IdxRidNextListPtr *nextIdxListPtr; IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; IdxRidListEntry newRowIdArray[MAX_BLOCK_ENTRY]; CommBlock cb; cb.file.oid = m_oid; cb.file.pFile = m_pFile; //This is the sub block info prevLbid = m_lbid; prevSbid = m_sbid; prevEntry = m_entry; prevPos = LIST_SUB_LLP_POS; preTotalBytes = SUBBLOCK_TOTAL_BYTES; if (prevLbid == m_hdrLbid) { if (m_hdrBlock.state >=BLK_READ) getSubBlockEntry(m_hdrBlock.data, m_sbid, prevPos, LIST_ENTRY_WIDTH, &lastSubIdxRidListPtr ); else return ERR_IDX_LIST_INVALID_DELETE; } else { if (m_curBlock.state >=BLK_READ) getSubBlockEntry(m_curBlock.data, m_sbid, prevPos, LIST_ENTRY_WIDTH, &lastSubIdxRidListPtr ); else return ERR_IDX_LIST_INVALID_DELETE; } found = false; m_lbid = ((IdxEmptyListEntry*)&lastSubIdxRidListPtr)->fbo; m_sbid = 0; m_entry = 0; type = lastSubIdxRidListPtr.type; count = lastSubIdxRidListPtr.count; pos = LIST_BLOCK_LLP_POS; totalbytes = BYTE_PER_BLOCK; //Not found in the first sub while ((!found) &&(type==LIST_BLOCK_TYPE)) { rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, totalbytes, rowIdArray); if (rc != NO_ERROR) return rc; m_curBlock.dirty = true; m_curBlock.state =BLK_READ; m_curBlock.lbid = m_lbid; prevType = type; //Save it just in case not found here lastIdxRidListPtr =(IdxRidListPtr *) &rowIdArray[pos]; type = lastIdxRidListPtr->type; count = lastIdxRidListPtr->count; //prepared for not found in current block //find out what is the next type //Next Type is needed here for (int i=0; i<count; i++) { if (rowIdArray[i].rid == rowId) {//found the rowid memcpy(&newRowIdArray[0],&rowIdArray[0],totalbytes); found = true; m_dLbid = m_lbid; m_dSbid = m_sbid; m_dEntry = i; lastIdxRidListPtr->count--; if (lastIdxRidListPtr->count==0) { if (!m_useNarray) { //get the previous value out, could be a sub block if (prevLbid == m_hdrLbid) getSubBlockEntry(m_hdrBlock.data, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr); else if (prevLbid == m_lbid) getSubBlockEntry(m_curBlock.data, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr); else rc = readSubBlockEntry(cb, &prevDataBlock, prevLbid, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr); if (rc != NO_ERROR) return rc; //check the type before set if (type == LIST_BLOCK_TYPE) { ((IdxEmptyListEntry*)&prevIdxRidListPtr)->fbo = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; ((IdxEmptyListEntry*)&prevIdxRidListPtr)->sbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->sbid; ((IdxEmptyListEntry*)&prevIdxRidListPtr)->entry = ((IdxEmptyListEntry*)lastIdxRidListPtr)->entry; //safety check prevIdxRidListPtr.type= type; } else // If no more links, the current one is gone also { if (prevIdxRidListPtr.count>0) { prevIdxRidListPtr.type =0; prevIdxRidListPtr.llp = 0; } else {//In case it is a sub block, not released with 0 count prevIdxRidListPtr.type =LIST_NOT_USED_TYPE; prevIdxRidListPtr.llp = 0; } }//end if type =LIST_SUBBLOCK_TYPE,LIST_BLOCK_TYPE //;set to LIST_NOT_USED_TYPE--unused before release lastIdxRidListPtr->type=LIST_NOT_USED_TYPE; lastIdxRidListPtr->llp =0; if (prevPos == LIST_BLOCK_LLP_POS) { if (prevLbid<m_lastLbid) rc = setLastLbid(prevLbid); } } } //end if count==0 else { memcpy(&rowIdArray[i],&newRowIdArray[i+1],(count-(i+1))*LIST_ENTRY_WIDTH); if (m_lastLbid > m_lbid) rc = setLastLbid(m_lbid); }//count check //Found rowId rowIdArray[count-1].type=LIST_NOT_USED_TYPE; rowIdArray[count-1].rid =0; m_curIdxRidListHdr.idxRidListSize.size--; //Write Out Put in another routine if ((prevLbid==m_hdrLbid) && (m_lbid != m_hdrLbid)) {// AAC --3 if (!m_useNarray) { if (lastIdxRidListPtr->count ==0) { setSubBlockEntry( m_hdrBlock.data, prevSbid, prevPos, width, &prevIdxRidListPtr ); } } setSubBlockEntry( m_curBlock.data, m_sbid, 0, totalbytes, rowIdArray ); setSubBlockEntry( m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr ); rc = writeDBFile( cb, m_hdrBlock.data, m_hdrLbid); if (rc != NO_ERROR) return rc; rc = writeDBFile( cb, m_curBlock.data, m_lbid); if (rc != NO_ERROR) return rc; m_hdrBlock.state = BLK_READ; m_curBlock.state = BLK_READ; } else { //ABC -- if (!m_useNarray) { if (lastIdxRidListPtr->count ==0) { setSubBlockEntry( prevDataBlock.data, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr ); rc = writeDBFile( cb, prevDataBlock.data,prevLbid); if (rc != NO_ERROR) return rc; } } setSubBlockEntry( m_curBlock.data, m_sbid, 0, totalbytes, rowIdArray ); setSubBlockEntry( m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr ); rc = writeDBFile( cb, m_hdrBlock.data, m_hdrLbid); if (rc != NO_ERROR) return rc; rc = writeDBFile( cb, m_curBlock.data, m_lbid); memset(m_hdrBlock.data,0, sizeof(m_hdrBlock.data)); if (rc != NO_ERROR) return rc; m_hdrBlock.state = BLK_READ; m_curBlock.state = BLK_READ; } //last case A B C --end 5 //Done with writing to disk // Now we need to release the segment if (!m_useNarray) { if (lastIdxRidListPtr->count ==0) { rc = releaseSegment(); if (rc != NO_ERROR) return rc; }// end release segment when count ==0 } m_entry =i; //for use in findRow ID return rc; //DONE !!!found then we return, no need to go on }//FOUND THE ROWID returned !!!! }//for loop i not found continue to i++ //NOT FOUND in this block go to next block //assigning the current llp as previous llp:lbid, sbid, entry prevLbid = m_lbid; prevSbid = 0; prevEntry = 0; prevPos = pos; preTotalBytes = totalbytes; m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; m_sbid = 0; m_entry = 0; }// end while if (!found) rc = ERR_IDX_LIST_INVALID_DELETE; return rc; }