/* TokenizeAll() - Get to the point we have a set of input tokens ready to parse.
*
* This function will, if it encounters no errors, completely tokenize all of the
* input of the designated input file, as well as all files it %include's.
* It will throw an exception on the first bad token (if any), but only after having
* tokenized all of the input, %include files as well.
****************************************************************************************/
static void TokenizeAll(unique_ref<TTokens> Tokens, string GrammarFilename,
int IncludeToken)
{
TFileTokens FileTokens;
TToken Token;
Tokens->Tokenize(GrammarFilename, IncludeToken);
FileTokens = Tokens->GetFile(GrammarFilename);
assert(FileTokens.Begin >= 0);
for(auto iToken=FileTokens.Begin; ; ++iToken)
{
Token = Tokens->Get(iToken);
if(Token.Type == TToken::TKEOF)
break;
if(Token.Type == TToken::INCLUDE)
{
Token = Tokens->Get(++iToken);
if(Token.Type != TToken::WHITESPACE)
Error("EXPECTING_WHITESPACE_AFTER_INCLUDE",
"Token", iToken);
Token = Tokens->Get(++iToken);
if(Token.Type != TToken::QUOTED)
Error("EXPECTING_QUOTED_STRING_AFTER_INCLUDE",
"Token", iToken);
auto Filename = Token.Unquote();
TokenizeAll(Tokens, Filename.get(), iToken);
}
}
}
/* ??? modify to handle includes */
TParseToken TParser::GetNextToken()
{
CurrentToken = Tokens->Get(TokenCursor++);
while( CurrentToken.Type == TToken::WHITESPACE
|| CurrentToken.Type == TToken::COMMENT
|| CurrentToken.Type == TToken::MCOMMENT
|| (IgnoreNewlines && CurrentToken.Type == TToken::NEWLINE))
CurrentToken = Tokens->Get(TokenCursor++);
CurrentToken.Index = TokenCursor-1;
if(CurrentToken.Type == TToken::TKEOF)
--TokenCursor;
return CurrentToken;
}
void FakeBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
std::unique_lock<std::mutex> lock(_mutex);
int numRemoved = _blocks.erase(key);
ASSERT(numRemoved == 1, "Block not found");
}
void OnDiskBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
OnDiskBlock::RemoveFromDisk(_rootdir, key);
}
void InMemoryBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
int numRemoved = _blocks.erase(key.ToString());
ASSERT(1==numRemoved, "Didn't find block to remove");
}
void CachingBlockStore::release(unique_ref<Block> block) {
Key key = block->key();
_cache.push(key, std::move(block));
}
blockstore::Key CreateBlockWriteFixtureToItAndReturnKey() {
auto block = blockStore->create(Data(data.size()));
block->write(data.data(), 0, data.size());
return block->key();
}
blockstore::Key CreateBlockReturnKey(const Data &initData) {
return blockStore->create(initData)->key();
}
OnDiskBlockCreateSizeTest():
block(OnDiskBlock::CreateOnDisk(dir.path(), key, std::move(Data(GetParam()).FillWithZeroes())).value()),
ZEROES(block->size())
{
ZEROES.FillWithZeroes();
}
void ParallelAccessDataTreeStore::remove(unique_ref<DataTreeRef> tree) {
Key key = tree->key();
return _parallelAccessStore.remove(key, std::move(tree));
}
unique_ref<DirBlob> DirBlob::InitializeEmptyDir(FsBlobStore *fsBlobStore, unique_ref<Blob> blob, std::function<off_t(const blockstore::Key&)> getLstatSize) {
InitializeBlob(blob.get(), FsBlobView::BlobType::DIR);
return make_unique_ref<DirBlob>(fsBlobStore, std::move(blob), getLstatSize);
}
void ParallelAccessFsBlobStore::remove(unique_ref<FsBlobRef> blob) {
Key key = blob->key();
return _parallelAccessStore.remove(key, std::move(blob));
}
void CachingFsBlobStore::remove(unique_ref<FsBlobRef> blob) {
auto baseBlob = blob->releaseBaseBlob();
return _baseBlobStore->remove(std::move(baseBlob));
}
