/**
* Searches through the pages loking for a page with enough space to store r. If no space
* is found, it grows the segment
*
* @param r: the record
*
* @return rtrn: TID identifying the location where r was stored
*/
TID SPSegment::insert(Record& r) {
TID rtrn;
SlottedPage* spHolder = NULL;
// instantiate new record pointer by given r
Record* newRec = new Record(r.getLen(), r.getData());
// try to find slotted page which can hold record
for (auto it = spMap.begin(); it != spMap.end(); ++it) {
if (it->second->getFreeSpace() >= (newRec->getLen() + (2 * sizeof(uint16_t)) + sizeof(bool))) {
spHolder = it->second;
rtrn.pageId = it->first;
break;
}
}
// page to hold record found
if (spHolder != NULL) {
// insert record into slotted page
rtrn.slotId = spHolder->insertRecord(*newRec);
// write changes back to disk
if (!writeToFrame(spHolder, rtrn.pageId)) {
cerr << "Cannot write slotted page into frame" << endl;
}
}
// no page found to hold record, so increase the segment
else {
// just need one more page
vector<uint64_t> neededExtents;
// grow by 10 pages
vector<uint64_t> newExtents = grow(10);
// create new slotted page
SlottedPage* sp = new SlottedPage();
sp->getHeader()->freeSpace = bm->getPageSize() - sizeof(Header);
rtrn.pageId = newExtents.front();
// insert record into slotted page
rtrn.slotId = sp->insertRecord(*newRec);
// write changes back to disk
if (writeToFrame(sp, rtrn.pageId)) {
spMap[rtrn.pageId] = sp;
} else {
cerr << "Cannot write slotted page into frame" << endl;
}
}
return rtrn;
}
TEST(SlottedPage, Simple)
{
SlottedPage* slottedPage = static_cast<SlottedPage*>(malloc(kPageSize));
slottedPage->initialize();
// Insert
slottedPage->insert(Record("windmill"));
RecordId fsi = slottedPage->insert(Record("windmill"));
slottedPage->remove(fsi);
free(slottedPage);
}
Record SPSegment::inPlaceLookup(TID tid) {
// TODO assert(tid.getPage() is part of this segment);
BufferFrame frame = bm->fixPage(tid.getPage(), false);
SlottedPage* page = reinterpret_cast<SlottedPage*>(frame.getData());
Slot* slot = page->getSlot(tid.getSlot());
if (slot->isMoved() || slot->isEmpty()) {
// Slot is empty: Return empty record
bm->unfixPage(frame, false);
return Record(0, nullptr);
} else {
// Slot has content: Return record with content.
bm->unfixPage(frame, false);
return Record(slot->length(), slot->getRecord()->getData());
}
}
bool SPSegment::remove(TID tid){
// TODO assert(tid.getPage() is part of this segment);
BufferFrame frame = bm->fixPage(tid.getPage(), true);
SlottedPage* page = reinterpret_cast<SlottedPage*>(frame.getData());
unsigned space = page->remove(tid.getSlot());
// Update FSI
this->fsi[tid.getPage()] = space;
bm->unfixPage(frame, true);
if (tid.tid == lastTID.tid) {
// TODO Update lastTID
}
return true;
}
/**
* Returns a pointer or reference to the readonly record associated with tid
*
* @param tid: the tuple ID
*
* @return rtrn: the record
*/
Record* SPSegment::lookup(TID tid) {
Record* rtrn;
try {
if (tid.pageId >= 0 && tid.slotId >= 0) {
// read slotted page out of memory map
if (spMap.count(tid.pageId) > 0) {
SlottedPage* sp = spMap.at(tid.pageId);
rtrn = sp->lookupRecord(tid.slotId);
}
// if not available read from frame
else {
// writes frame into map too
SlottedPage* sp = readFromFrame(tid.pageId);
if (sp != NULL) {
rtrn = sp->lookupRecord(tid.slotId);
} else {
throw invalid_argument("No page found by given tid");
}
}
} else {
throw invalid_argument("Given tid is invalid");
}
// in case of record is a redirection, lookup for redirected data record
if (!rtrn->isDataRecord()) {
rtrn = lookup(rtrn->redirection);
}
} catch (const invalid_argument& e) {
cerr << "Invalid argument @lookup segment: " << e.what() << endl;
rtrn = NULL;
}
return rtrn;
}
/*
* Reorders data of not removed slots to get free space
* Returns free space in bytes after defragmentation
*/
uint16_t SlottedPage::defrag(){
SlottedPage spWorkingCopy;
spWorkingCopy.header.slotCount = 0;
spWorkingCopy.header.dataStart = PAGESIZE - sizeof(SlottedPage::SPHeader);
spWorkingCopy.header.fragmentedSpace = 0;
spWorkingCopy.header.numUnusedSlots = 0;
for(uint16_t i = 0; i < header.slotCount; ++i){
if(!isRemoved(i) && !isIndirection(i)){
Slot* s = &slots[i];
uint16_t wsID = spWorkingCopy.insertNewSlot(&data[s->offset], s->length);
s->offset = spWorkingCopy.slots[wsID].offset;
}
}
//data is now sorted at the back -> update our SlottedPage
header.dataStart = spWorkingCopy.header.dataStart;
memcpy(&data[header.dataStart], &spWorkingCopy.data[spWorkingCopy.header.dataStart], PAGESIZE - sizeof(SPHeader) - spWorkingCopy.header.dataStart);
header.fragmentedSpace = 0;
return getFreeSpaceInBytes();
}
TEST(SlottedPage, ReferenceRecords)
{
SlottedPage* slottedPage = static_cast<SlottedPage*>(malloc(kPageSize));
slottedPage->initialize();
TupleId tid = TupleId(8129);
RecordId rid = slottedPage->insert(Record("most awesome paper ever: a system for visualizing human behavior based on car metaphors"));
// Make reference and check
slottedPage->updateToReference(rid, tid);
ASSERT_EQ(tid, slottedPage->isReference(rid));
ASSERT_EQ(slottedPage->getAllRecords(kInvalidPageId).size(), 0u);
ASSERT_EQ(slottedPage->countAllRecords(), 1u);
// Remove reference
slottedPage->remove(rid);
ASSERT_EQ(slottedPage->getAllRecords(kInvalidPageId).size(), 0u);
ASSERT_EQ(slottedPage->countAllRecords(), 0u);
free(slottedPage);
}
TID SPSegment::insert(const Record& r){
// Find page with enough space for r
uint64_t pageId = this->lastPage + 1;
for (auto it = this->fsi.rbegin(); it != this->fsi.rend(); it++) {
if (it->second >= r.getLen()) {
pageId = it->first;
break;
}
}
// If necessary, create new SlottedPage
BufferFrame frame = bm->fixPage(pageId, true);
SlottedPage* page = reinterpret_cast<SlottedPage*>(frame.getData());
if (pageId > this->lastPage) {
*page = SlottedPage();
this->lastPage++;
if (lastPage > 1l << 48) throw "Max page number reached.";
}
// TODO Reorder record ?
// Write to page
unsigned slotNum = page->insert(r);
bm->unfixPage(frame, true);
// Update lastTID
TID newTID = TID(pageId, slotNum);
if (newTID.tid > this->lastTID.tid) {
this->lastTID = newTID;
}
// Update FSI
this->fsi[newTID.getPage()] -= r.getLen() + (slotNum == page->getMaxSlot() ? sizeof(Slot) : 0);
return newTID;
}
Record SPSegment::lookup(TID tid) {
// TODO assert(tid.getPage() is part of this segment);
BufferFrame frame = bm->fixPage(tid.getPage(), false);
SlottedPage* page = reinterpret_cast<SlottedPage*>(frame.getData());
Slot* slot = page->getSlot(tid.getSlot());
if (slot->isMoved()) {
// Slot was indirected: Lookup that TID the slot points to recursively.
bm->unfixPage(frame, false);
return this->lookup(TID(slot->slot));
} else if (slot->length() == 0 && slot->offset() == 0) {
// Slot is empty: Return empty record
bm->unfixPage(frame, false);
return Record(0, nullptr);
} else {
// Slot has content: Return record with content.
bm->unfixPage(frame, false);
return Record(slot->length(), slot->getRecord()->getData());
}
}
/**
* Deletes the record pointed to by tid
*
* @param tid: the tuple ID
*
* @return rtrn: whether successfully or not
*/
bool SPSegment::remove(TID tid) {
bool rtrn = true;
try {
if (tid.pageId >= 0 && tid.slotId >= 0) {
if (spMap.count(tid.pageId) > 0) {
SlottedPage* sp = spMap.at(tid.pageId);
Record* recordToRemove = sp->lookupRecord(tid.slotId);
// in case of record to remove is a redirection, first remove remote record
if (!recordToRemove->isDataRecord()) {
remove(recordToRemove->redirection);
}
sp->removeRecord(tid.slotId);
// write changes back to disk
if (!writeToFrame(sp, tid.pageId)) {
cerr << "Cannot write slotted page into frame" << endl;
}
}
} else {
throw invalid_argument("Given tid is invalid");
}
} catch (const invalid_argument& e) {
cerr << "Invalid argument @remove segment: " << e.what() << endl;
rtrn = false;
}
return rtrn;
}
/**
* Constructor: initializes slotted pages
*/
SPSegment::SPSegment(vector<uint64_t> freeExtents, uint64_t segId, FSISegment *fsi,BufferManager * bm) :
Segment(freeExtents, segId, fsi, bm) {
for (unsigned i = 0; i < getSize(); i++) {
SlottedPage* sp = new SlottedPage();
sp->getHeader()->freeSpace = bm->getPageSize() - sizeof(Header);
uint64_t pageId = at(i);
// write slottet page to disk
if (writeToFrame(sp, pageId)) {
spMap.insert(make_pair(pageId, sp));
} else {
cerr << "Cannot write slotted page into frame" << endl;
}
}
cout << "SPSegment extents: " << std::endl;
for (unsigned i = 0; i < extents.size(); ++i) {
std::cout << "value: " << i << ": " << extents.at(i) << std::endl;
}
}
TEST(SlottedPage, SlotReuseAfterDelete)
{
SlottedPage* slottedPage = static_cast<SlottedPage*>(malloc(kPageSize));
slottedPage->initialize();
// Checks if a slot is reused
RecordId dataRecordId1 = slottedPage->insert(Record("Hello World!"));
slottedPage->remove(dataRecordId1);
RecordId dataRecordId2 = slottedPage->insert(Record("Hello World!"));
ASSERT_EQ(dataRecordId1, dataRecordId2);
// Even with smaller vales ?
slottedPage->remove(dataRecordId2);
RecordId dataRecordId = slottedPage->insert(Record("Hello World"));
ASSERT_EQ(dataRecordId, dataRecordId2);
free(slottedPage);
}
int main() {
char *block = (char *) malloc((size_t) BLOCKSIZE);
SlottedPage *sp = new SlottedPage(block);
sp->initBlock();
cout << "A fresh block just after initialization" << endl;
sp->display();
int opcode;
short slotNum;
char *rec = (char *) malloc(MAX_INPUT_REC_SIZE);
cout << "1: Insert, 2: Get, 3: Delete, 4: Display, 5: Quit \n";
while (1) {
scanf("%d", &opcode);
switch (opcode) {
case INSERT:
scanf("%s", rec);
sp->storeRecord(rec);
break;
case GET:
scanf("%hu", &slotNum);
cout << sp->getRecord(slotNum) << endl;
break;
case DELETE:
scanf("%hu", &slotNum);
sp->deleteRecord(slotNum);
break;
case DISPLAY:
sp->display();
break;
case QUIT:
return 0;
default:
cout << "1: Insert, 2: Get, 3: Delete, 4: Display, 5: Quit \n";
}
}
return 0;
}
TEST(SlottedPage, ForeignRecords)
{
SlottedPage* slottedPage = static_cast<SlottedPage*>(malloc(kPageSize));
slottedPage->initialize();
// Make foreign record and check
uint16_t freeBytes = slottedPage->getBytesFreeForRecord();
RecordId rid = slottedPage->insertForeigner(Record("fear not this night"), TupleId(8129));
ASSERT_EQ(slottedPage->lookup(rid), Record("fear not this night"));
ASSERT_EQ(slottedPage->isReference(rid), kInvalidTupleId);
ASSERT_EQ(1u, slottedPage->getAllRecords(kInvalidPageId).size());
ASSERT_EQ(TupleId(8129), slottedPage->getAllRecords(kInvalidPageId)[0].first);
ASSERT_EQ(Record("fear not this night"), slottedPage->getAllRecords(kInvalidPageId)[0].second);
// Update the foreign record
slottedPage->updateForeigner(rid, TupleId(8129), Record("but i am afraid of the dark"));
ASSERT_EQ(slottedPage->lookup(rid), Record("but i am afraid of the dark"));
ASSERT_EQ(slottedPage->isReference(rid), kInvalidTupleId);
ASSERT_EQ(1u, slottedPage->getAllRecords(kInvalidPageId).size());
ASSERT_EQ(TupleId(8129), slottedPage->getAllRecords(kInvalidPageId)[0].first);
ASSERT_EQ(Record("but i am afraid of the dark"), slottedPage->getAllRecords(kInvalidPageId)[0].second);
// Remove foreign record
slottedPage->remove(rid);
ASSERT_EQ(slottedPage->getBytesFreeForRecord(), freeBytes);
ASSERT_EQ(slottedPage->countAllRecords(), 0u);
free(slottedPage);
}
TEST(SlottedPage, DefragmentationBasic)
{
std::string fragmentationData1 = "Hello !";
std::string fragmentationData2 = "World !";
std::string staticData = "the blob";
std::string newData = "hello my world !";
Record fragmentationRecord1(fragmentationData1);
Record fragmentationRecord2(fragmentationData2);
Record staticRecord(staticData);
Record newDataRecord(newData);
SlottedPage* slottedPage = static_cast<SlottedPage*>(malloc(kPageSize));
slottedPage->initialize();
RecordId fragmentationRecordId1 = slottedPage->insert(fragmentationRecord1);
RecordId staticRecordId = slottedPage->insert(staticRecord);
RecordId fragmentationRecordId2 = slottedPage->insert(fragmentationRecord2);
// Fill page with ... CRAP!
while(slottedPage->canHoldRecord(staticRecord))
slottedPage->insert(Record(staticData));
// Remove the two values
ASSERT_TRUE(!slottedPage->canHoldRecord(newDataRecord));
slottedPage->remove(fragmentationRecordId1);
slottedPage->remove(fragmentationRecordId2);
ASSERT_TRUE(slottedPage->getBytesFreeForRecord() >= newData.length());
// The page should now have a structure like: <freeSpace> <staticData> <freeSpace> (whereby both free space areas are to short for the new record)
slottedPage->defragment();
// No side effects on sample record
ASSERT_EQ(slottedPage->isReference(staticRecordId), kInvalidTupleId);
ASSERT_EQ(slottedPage->lookup(staticRecordId), Record(staticData));
// Add now record
ASSERT_TRUE(slottedPage->getBytesFreeForRecord() >= newData.length());
RecordId newDataRecordId = slottedPage->insert(newDataRecord);
ASSERT_EQ(slottedPage->isReference(newDataRecordId), kInvalidTupleId);
ASSERT_EQ(slottedPage->lookup(newDataRecordId), newDataRecord);
free(slottedPage);
}
bool SPSegment::update(TID tid, const Record& r){
Record r_old = this->lookup(tid);
unsigned len_old = r_old.getLen();
unsigned len_new = r.getLen();
BufferFrame frame = bm->fixPage(tid.getPage(), true);
SlottedPage* page = reinterpret_cast<SlottedPage*>(frame.getData());
if(len_old == len_new){
// If size doesn't change, use memcpy
memcpy(page->getSlot(tid.getSlot())->getRecord(), &r, r.getLen());
} else if(len_old > len_new){
// Record has become smaller
memcpy(page->getSlot(tid.getSlot())->getRecord(), &r, r.getLen());
// TODO Update freeSpace of page
// TODO update FSI
} else {
// Record has become larger
unsigned freeSpaceOnPage = page->remove(tid.getSlot());
this->fsi[tid.getPage()] = freeSpaceOnPage;
if (freeSpaceOnPage >= len_new) {
// It fits on the page after removal
page->insert(r);
} else {
// Even after removal it is too large
// Get another page
uint64_t sndPageId = this->lastPage + 1;
for (auto it = this->fsi.rbegin(); it != this->fsi.rend(); it++) {
if (it->second >= r.getLen()) {
sndPageId = it->first;
break;
}
}
BufferFrame sndFrame = bm->fixPage(sndPageId, true);
// Create a new SlottedPage if necessary
SlottedPage* sndPage = reinterpret_cast<SlottedPage*>(sndFrame.getData());
if (sndPageId > this->lastPage) {
*sndPage = SlottedPage();
this->lastPage++;
if (lastPage > 1l << 48) throw "Max page number reached.";
}
Slot* fstSlot = page->getSlot(tid.getSlot());
assert(fstSlot->isEmpty());
// Insert into new page
unsigned sndSlotNum = sndPage->insert(r);
this->fsi[sndPageId] -= r.getLen() + (sndSlotNum == sndPage->getMaxSlot() ? sizeof(Slot) : 0);
// Update first slot to directo to second page.
*fstSlot = Slot(TID(sndPageId, sndSlotNum));
bm->unfixPage(sndFrame, true);
}
}
bm->unfixPage(frame, true);
return true;
}
TEST(SlottedPage, Randomized)
{
const uint32_t kTestScale = 1;
const uint32_t iterations = 10000;
util::Random ranny;
for(uint32_t j=0; j<kTestScale; j++) {
std::unordered_map<RecordId, std::string> reference;
SlottedPage* slottedPage = static_cast<SlottedPage*>(malloc(kPageSize));
slottedPage->initialize();
// Add some initial data
for(uint32_t i=0; i<kPageSize/3/32; i++) {
std::string data = util::randomWord(ranny, 8, 64);
if(slottedPage->getBytesFreeForRecord() < data.size())
continue;
RecordId id = slottedPage->insert(Record(data));
// std::cout << "insert " << id << " -> " << data << std::endl;
ASSERT_TRUE(reference.count(id) == 0);
reference.insert(make_pair(id, data));
}
// Work on it
for(uint32_t i=0; i<iterations; i++) {
int32_t operation = ranny.rand() % 100;
// Do insert
if(operation <= 40) {
std::string data = util::randomWord(ranny, 8, 64);
if(slottedPage->getBytesFreeForRecord() < data.size())
continue;
RecordId id = slottedPage->insert(Record(data));
// std::cout << "insert " << id << " -> " << data << std::endl;
ASSERT_TRUE(reference.count(id) == 0);
reference.insert(make_pair(id, data));
}
// Do remove
else if(operation <= 80) {
if(reference.empty())
continue;
RecordId id = reference.begin()->first;
// std::cout << "remove " << id << std::endl;
Record record = slottedPage->lookup(id);
ASSERT_EQ(slottedPage->isReference(id), kInvalidTupleId);
ASSERT_EQ(std::string(record.data(), record.size()), reference.begin()->second);
slottedPage->remove(id);
reference.erase(reference.begin());
}
// Do update
else if(operation <= 98) {
if(reference.empty())
continue;
RecordId id = reference.begin()->first;
Record record = slottedPage->lookup(id);
ASSERT_EQ(slottedPage->isReference(id), kInvalidTupleId);
ASSERT_EQ(std::string(record.data(), record.size()), reference.begin()->second);
std::string data = util::randomWord(ranny, 8, 64);
if(slottedPage->canUpdateRecord(id, Record(data))) {
slottedPage->update(id, Record(data));
reference.erase(reference.begin());
reference.insert(make_pair(id, data));
}
}
// Do consistency check
else if(operation<=99 || i==iterations-1 || i==0) {
ASSERT_TRUE(slottedPage->isValid());
auto records = slottedPage->getAllRecords(kInvalidPageId); // page id does not matter
ASSERT_EQ(records.size(), reference.size());
for(auto& iter : records) {
ASSERT_TRUE(reference.count(iter.first.toRecordId()) > 0);
ASSERT_EQ(string(iter.second.data(), iter.second.size()), reference.find(iter.first.toRecordId())->second);
}
continue;
}
}
free(slottedPage);
}
}
/**
* Updates the record pointed to by tid with the content of record r
*
* @param tid: the tuple ID
* @param r: the record
*
* @return rtrn: whether successfully or not
*/
bool SPSegment::update(TID tid, Record& r) {
bool rtrn = true;
try {
if (tid.pageId >= 0 && tid.slotId >= 0) {
if (spMap.count(tid.pageId) > 0) {
// fetch slotted page
SlottedPage* sp = spMap.at(tid.pageId);
// slotted page which must be updated
SlottedPage* spUpdate;
if (sp->getRecordsMap().count(tid.slotId) > 0) {
// fetch old record
Record* rOld = sp->getRecordsMap().at(tid.slotId);
// record which must be updated
Record* rUpdate;
// tid which must be updated
TID tidUpdate;
if (rOld->isDataRecord()) {
// in case of old record is a data record, update the old record and slottedpage
rUpdate = rOld;
spUpdate = sp;
tidUpdate = tid;
} else {
// in case of old record is a redirection, update the corresponding remote record and slotted page
rUpdate = lookup(rOld->redirection);
spUpdate = spMap.at(rOld->redirection.pageId);
tidUpdate = rOld->redirection;
}
// calculate whether or not the sp has enough free space to contain the new record
bool enoughFreeSpace = (spUpdate->getFreeSpace() + rUpdate->getLen()) >= r.getLen();
// if there is enough free space just update
if (enoughFreeSpace) {
spUpdate->updateRecord(tidUpdate.slotId, r);
// write changes back to disk
if (!writeToFrame(spUpdate, tidUpdate.pageId)) {
cerr << "Cannot write slotted page into frame" << endl;
}
}
// otherwise redirect record
else {
// 1st step: insert record into new page with enough space
TID redTid = insert(r);
// 2nd step: write/update redirection into old position
rOld->dataRecord = false;
rOld->data = 0;
rOld->len = 0;
rOld->redirection = redTid;
// 3rd step: update header information for slotted page of rOld
sp->updateRecord(tid.slotId, *rOld);
// write changes back to disk
if (!writeToFrame(sp, tid.pageId)) {
cerr << "Cannot write slotted page into frame" << endl;
}
}
} else {
throw invalid_argument("No record found by given slotId");
}
} else {
throw invalid_argument("No page found by given tid");
}
} else {
throw invalid_argument("Given tid is invalid");
}
} catch (const invalid_argument& e) {
cerr << "Invalid argument @lookup segment: " << e.what() << endl;
rtrn = false;
}
return rtrn;
}
