c4_FormatB::c4_FormatB (const c4_Property& prop_, c4_HandlerSeq& seq_)
: c4_FormatHandler (prop_, seq_), _data (seq_.Persist()),
_sizeCol (seq_.Persist()), _memoCol (seq_.Persist()), _recalc (false)
{
_offsets.SetSize(1, 100);
_offsets.SetAt(0, 0);
}
c4_HandlerSeq::c4_HandlerSeq(c4_HandlerSeq &owner_, c4_Handler *handler_):
_persist(owner_.Persist()), _field(owner_.FindField(handler_)), _parent
(&owner_), _numRows(0)
{
for (int i = 0; i < NumFields(); ++i) {
c4_Field &field = Field(i);
c4_Property prop(field.Type(), field.Name());
d4_dbgdef(int n =)AddHandler(f4_CreateFormat(prop, *this));
d4_assert(n == i);
}
}
void c4_SaveContext::CommitSequence(c4_HandlerSeq &seq_, bool selfDesc_) {
StoreValue(0); // sias prefix
if (selfDesc_) {
c4_String desc = seq_.Description();
int k = desc.GetLength();
StoreValue(k);
Write((const char*)desc, k);
}
StoreValue(seq_.NumRows());
if (seq_.NumRows() > 0)
for (int i = 0; i < seq_.NumFields(); ++i)
seq_.NthHandler(i).Commit(*this);
}
c4_FormatV::c4_FormatV(const c4_Property &prop_, c4_HandlerSeq &seq_):
c4_FormatHandler(prop_, seq_), _data(seq_.Persist()), _inited(false){}
c4_FormatX::c4_FormatX(const c4_Property &p_, c4_HandlerSeq &s_, int w_):
c4_FormatHandler(p_, s_), _data(s_.Persist(), w_){}
void c4_SaveContext::SaveIt(c4_HandlerSeq &root_, c4_Allocator **spacePtr_,
c4_Bytes &rootWalk_) {
d4_assert(_space != 0);
const t4_i32 size = _strategy.FileSize();
if (_strategy._failure != 0)
return ;
const t4_i32 end = _fullScan ? 0 : size - _strategy._baseOffset;
if (_differ == 0) {
if (_mode != 1)
_space->Initialize();
// don't allocate anything inside the file in extend mode
if (_mode == 2 && end > 0) {
_space->Occupy(1, end - 1);
_nextSpace->Occupy(1, end - 1);
}
// the header is always reserved
_space->Occupy(1, 7);
_nextSpace->Occupy(1, 7);
if (end > 0) {
d4_assert(end >= 16);
_space->Occupy(end - 16, 16);
_nextSpace->Occupy(end - 16, 16);
_space->Occupy(end, 8);
_nextSpace->Occupy(end, 8);
}
}
//AllocDump("a1", false);
//AllocDump("a2", true);
// first pass allocates columns and constructs shallow walks
c4_Column walk(root_.Persist());
SetWalkBuffer(&walk);
CommitSequence(root_, true);
SetWalkBuffer(0);
CommitColumn(walk);
c4_Bytes tempWalk;
walk.FetchBytes(0, walk.ColSize(), tempWalk, true);
t4_i32 limit = _nextSpace->AllocationLimit();
d4_assert(limit >= 8 || _differ != 0);
if (limit < 0) {
// 2006-01-12 #2: catch file size exceeding 2 Gb
_strategy._failure = - 1; // unusual non-zero value flags this case
return ;
}
bool changed = _fullScan || tempWalk != rootWalk_;
rootWalk_ = c4_Bytes(tempWalk.Contents(), tempWalk.Size(), true);
_preflight = false;
// special-case to avoid saving data if file is logically empty
// in that case, the data is 0x80 0x81 0x80 (plus the header)
if (!_fullScan && limit <= 11 && _differ == 0) {
_space->Initialize();
_nextSpace->Initialize();
changed = false;
}
if (!changed)
return ;
//AllocDump("b1", false);
//AllocDump("b2", true);
if (_differ != 0) {
int n = _differ->NewDiffID();
_differ->CreateDiff(n, walk);
return ;
}
d4_assert(_mode != 0 || _fullScan);
// this is the place where writing may start
// figure out where the new file ends and write a skip tail there
t4_i32 end0 = end;
// true if the file need not be extended due to internal free space
bool inPlace = end0 == limit - 8;
if (inPlace) {
d4_assert(!_fullScan);
_space->Release(end0, 8);
_nextSpace->Release(end0, 8);
end0 -= 16; // overwrite existing tail markers
} else {
/* 18-11-2005 write new end marker and flush it before *anything* else! */
if (!_fullScan && end0 < limit) {
c4_FileMark mark1(limit, 0);
_strategy.DataWrite(limit, &mark1, sizeof mark1);
_strategy.DataCommit(0);
if (_strategy._failure != 0)
return ;
}
c4_FileMark head(limit + 16-end, _strategy._bytesFlipped, end > 0);
_strategy.DataWrite(end, &head, sizeof head);
if (end0 < limit)
end0 = limit;
// create a gap
}
t4_i32 end1 = end0 + 8;
t4_i32 end2 = end1 + 8;
if (!_fullScan && !inPlace) {
c4_FileMark mark1(end0, 0);
_strategy.DataWrite(end0, &mark1, sizeof mark1);
#if q4_WIN32
/* March 8, 2002
* On at least NT4 with NTFS, extending a file can cause it to be
* rounded up further than expected. To prevent creating a bad
* file (since the file does then not end with a marker), the
* workaround it so simply accept the new end instead and rewrite.
* Note that between these two writes, the file is in a bad state.
*/
t4_i32 realend = _strategy.FileSize() - _strategy._baseOffset;
if (realend > end1) {
end0 = limit = realend - 8;
end1 = realend;
end2 = realend + 8;
c4_FileMark mark1a(end0, 0);
_strategy.DataWrite(end0, &mark1a, sizeof mark1a);
}
#endif
d4_assert(_strategy.FileSize() == _strategy._baseOffset + end1);
}
_space->Occupy(end0, 16);
_nextSpace->Occupy(end0, 16);
// strategy.DataCommit(0); // may be needed, need more info on how FS's work
// but this would need more work, since we can't adjust file-mapping here
// second pass saves the columns and structure to disk
CommitSequence(root_, true); // writes changed columns
CommitColumn(walk);
//! d4_assert(_curr == 0);
d4_assert(_nextPosIndex == _newPositions.GetSize());
if (_fullScan) {
c4_FileMark mark1(limit, 0);
_strategy.DataWrite(_strategy.FileSize() - _strategy._baseOffset, &mark1,
sizeof mark1);
c4_FileMark mark2(limit - walk.ColSize(), walk.ColSize());
_strategy.DataWrite(_strategy.FileSize() - _strategy._baseOffset, &mark2,
sizeof mark2);
return ;
}
if (inPlace)
d4_assert(_strategy.FileSize() == _strategy._baseOffset + end2);
else {
// make sure the allocated size hasn't changed
d4_assert(_nextSpace->AllocationLimit() == limit + 16);
d4_assert(end0 >= limit);
d4_assert(_strategy.FileSize() - _strategy._baseOffset == end1);
}
if (walk.Position() == 0 || _strategy._failure != 0)
return ;
_strategy.DataCommit(0);
c4_FileMark mark2(walk.Position(), walk.ColSize());
_strategy.DataWrite(end1, &mark2, sizeof mark2);
d4_assert(_strategy.FileSize() - _strategy._baseOffset == end2);
// do not alter the file header in extend mode, unless it is new
if (!_fullScan && (_mode == 1 || end == 0)) {
_strategy.DataCommit(0);
c4_FileMark head(end2, _strategy._bytesFlipped, false);
d4_assert(head.IsHeader());
_strategy.DataWrite(0, &head, sizeof head);
// if the file became smaller, we could shrink it
if (limit + 16 < end0) {
/*
Not yet, this depends on the strategy class being able to truncate, but
there is no way to find out whether it does (the solution is to write tail
markers in such a way that the file won't grow unnecessarily if it doesn't).
The logic will probably be:
* write new skip + commit "tails" at limit (no visible effect on file)
* overwrite commit tail at end with a skip to this new one (equivalent)
* replace header with one pointing to that internal new one (equivalent)
* flush (now the file is valid both truncated and not-yet-truncated
end = limit;
*/
}
}
// if using memory mapped files, make sure the map is no longer in use
if (_strategy._mapStart != 0)
root_.UnmappedAll();
// commit and tell strategy object what the new file size is, this
// may be smaller now, if old data at the end is no longer referenced
_strategy.DataCommit(end2);
d4_assert(_strategy.FileSize() - _strategy._baseOffset == end2);
if (spacePtr_ != 0 && _space != _nextSpace) {
d4_assert(*spacePtr_ == _space);
delete *spacePtr_;
*spacePtr_ = _nextSpace;
_nextSpace = 0;
}
}