void BufferAccessor::Iterate(CodeGen &codegen, llvm::Value *buffer_ptr,
BufferAccessor::IterateCallback &callback) const {
auto *start = codegen.Load(BufferProxy::buffer_start, buffer_ptr);
auto *end = codegen.Load(BufferProxy::buffer_pos, buffer_ptr);
lang::Loop loop{codegen, codegen->CreateICmpNE(start, end), {{"pos", start}}};
{
auto *pos = loop.GetLoopVar(0);
// Read
std::vector<codegen::Value> vals;
UpdateableStorage::NullBitmap null_bitmap(codegen, storage_format_, pos);
for (uint32_t col_id = 0; col_id < storage_format_.GetNumElements();
col_id++) {
auto val = storage_format_.GetValue(codegen, pos, col_id, null_bitmap);
vals.emplace_back(val);
}
// Invoke callback
callback.ProcessEntry(codegen, vals);
// Move along
auto *next = codegen->CreateConstInBoundsGEP1_64(
pos, storage_format_.GetStorageSize());
loop.LoopEnd(codegen->CreateICmpNE(next, end), {next});
}
}
void Sorter::Init(CodeGen &codegen, llvm::Value *sorter_ptr,
llvm::Value *executor_ctx,
llvm::Value *comparison_func) const {
auto *tuple_size = codegen.Const32(storage_format_.GetStorageSize());
codegen.Call(SorterProxy::Init,
{sorter_ptr, executor_ctx, comparison_func, tuple_size});
}
void Parser::ExprTail(ExprRec& result) {
ExprRec leftOperand, rightOperand;
OpRec op;
switch (NextToken()) {
case PLUS_OP:
case MINUS_OP:
leftOperand.kind = result.kind;
leftOperand.ival = result.ival; /* Only allowed for */
leftOperand.s_fval = result.s_fval; /* INTs and FLOATs */
leftOperand.name = result.name;
leftOperand.var_type = result.var_type;
AddOp();
code.ProcessOp(op); /*** CODE ***/
Factor(rightOperand);
/*** CODE ***/
code.GenInfix(leftOperand, op, rightOperand, result);
ExprTail(result);
break;
case RSTAPLE:
case RBANANA:
case RMUSTACHE:
case SEMICOLON:
case COMMA:
case LT_OP:
case LE_OP:
case GT_OP:
case GE_OP:
case EQ_OP1:
case EQ_OP2:
case NE_OP:
break;
default:
SyntaxError(NextToken(), "Unknown ExpressionTail Symbol.");
}
}
void Sorter::SortTopKParallel(CodeGen &codegen, llvm::Value *sorter_ptr,
llvm::Value *thread_states,
uint32_t sorter_offset, uint64_t top_k) const {
auto *offset = codegen.Const32(sorter_offset);
codegen.Call(SorterProxy::SortTopKParallel,
{sorter_ptr, thread_states, offset, codegen.Const64(top_k)});
}
// Iterate over the tuples in the sorter in batches/vectors of the given size
void Sorter::VectorizedIterate(
CodeGen &codegen, llvm::Value *sorter_ptr, uint32_t vector_size,
Sorter::VectorizedIterateCallback &callback) const {
llvm::Value *start_pos = GetStartPosition(codegen, sorter_ptr);
llvm::Value *num_tuples = GetNumberOfStoredTuples(codegen, sorter_ptr);
// Determine the number of bytes to skip per vector
llvm::Value *vec_sz = codegen.Const32(vector_size);
llvm::Value *tuple_size = GetTupleSize(codegen);
llvm::Value *skip = codegen->CreateMul(vec_sz, tuple_size);
lang::VectorizedLoop loop{
codegen, num_tuples, vector_size, {{"pos", start_pos}}};
{
llvm::Value *curr_pos = loop.GetLoopVar(0);
auto curr_range = loop.GetCurrentRange();
// Provide an accessor into the sorted space
SorterAccess sorter_access{*this, start_pos};
// Issue the callback
callback.ProcessEntries(codegen, curr_range.start, curr_range.end,
sorter_access);
// Bump the pointer by the size of a tuple
llvm::Value *next_pos = codegen->CreateInBoundsGEP(curr_pos, skip);
loop.LoopEnd(codegen, {next_pos});
}
}
CodeGen hex(AST &ast){
CodeGen cg;
cg.generate(ast.get_root());
cg.back_patch();
cg.fill();
// cg.print_hex();
// cg.print_tables();
return cg;
}
void Parser::VarList()
{
ExprRec var, index;
/* Listen to the first variable */
Variable(var, index);
code.ProcessVar(var, index); /*** CODE ***/
code.Listen(var, index); /*** CODE ***/
/* Listen for the next variables */
VarListTail(var, index);
}
void testCodeGen(){
CodeGen mCode;
;
//testFile(mCode.genOPENATLAS_NS_RESOURCE_CLASS());
testFile(mCode.genOPENATLAS_NS_RESOURCE_CLASS_PUBLIC_FUNCTION_IMPL());
// mCode.genSetterFuntion("drawable", CodeGen::FUN_TYPE_FUNCTION_GET);
// mCode.genGetterFunction("drawable", "icon");
}
/*
classRecogniser( key )( object ) -> Bool
*/
Ref * sysClassRecogniser( Ref * pc, MachineClass *vm ) {
if ( vm->count != 1 ) throw Ginger::Mishap( "Wrong number of arguments" );
Ref kk = vm->fastPeek();
if ( !IsObj( kk ) || *RefToPtr4( kk ) != sysClassKey ) throw Ginger::Mishap( "Key needed" );
CodeGen codegen = vm->codegen();
codegen->vmiFUNCTION( makeName( "is", RefToPtr4( kk )[ CLASS_OFFSET_TITLE ] ), 1, 1 );
codegen->vmiSYS_CALL_ARG( sysargRecognise, kk );
codegen->vmiSYS_RETURN();
vm->fastPeek() = codegen->vmiENDFUNCTION();
return pc;
}
void BufferAccessor::Append(CodeGen &codegen, llvm::Value *buffer_ptr,
const std::vector<codegen::Value> &tuple) const {
auto *size = codegen.Const32(storage_format_.GetStorageSize());
auto *space = codegen.Call(BufferProxy::Append, {buffer_ptr, size});
// Now, individually store the attributes of the tuple into the free space
UpdateableStorage::NullBitmap null_bitmap(codegen, storage_format_, space);
for (uint32_t col_id = 0; col_id < tuple.size(); col_id++) {
storage_format_.SetValue(codegen, space, col_id, tuple[col_id],
null_bitmap);
}
null_bitmap.WriteBack(codegen);
}
void Parser::IfStmt() {
ExprRec result;
Match(IF_SYM);
Match(LBANANA);
Condition(result);
Match(RBANANA);
code.IfThen(result);
StmtList();
ElseClause();
Match(END_SYM);
code.IfEnd();
}
void Parser::LoopStmt() {
ExprRec result;
Match(DO_SYM);
code.LoopBegin();
StmtList();
Match(UNTIL_SYM);
Match(LBANANA);
Condition(result);
Match(RBANANA);
code.LoopEnd(result);
Match(SEMICOLON);
}
void Parser::WhileStmt() {
ExprRec result;
Match(WHILE_SYM);
Match(LBANANA);
code.WhileTag(); /* TODO that is a new action symbol - add to grammar */
Condition(result);
Match(RBANANA);
code.WhileBegin(result);
StmtList();
Match(END_SYM);
code.WhileEnd();
}
void Parser::ForAssign()
{
/* Variable to be assigned a value */
ExprRec var, index;
Variable(var, index);
code.ProcessVar(var, index);
/* Equal sign '=' */
Match(ASSIGN_OP);
/* Value/Expression to assign to the variable */
ExprRec result;
Expression(result);
code.ForAssign(var, index, result);
}
void Parser::AssignStmt()
{
/* Variable to be assigned a value */
ExprRec var, index;
Variable(var, index);
code.ProcessVar(var, index);
/* Equal sign '=' */
Match(ASSIGN_OP);
/* Value/Expression to assign to the variable */
ExprRec result;
AssignTail(result);
code.Assign(var, index, result);
Match(SEMICOLON);
}
/*
classConstructor( key )( arg1, ..., argN ) -> instance
*/
Ref * sysClassConstructor( Ref * pc, MachineClass *vm ) {
if ( vm->count != 1 ) throw Ginger::Mishap( "Wrong number of arguments" );
Ref kk = vm->fastPeek();
if ( !IsObj( kk ) || *RefToPtr4( kk ) != sysClassKey ) throw Ginger::Mishap( "Key needed" );
Ref * obj_K = RefToPtr4( kk );
long n = SmallToLong( obj_K[ CLASS_OFFSET_NFIELDS ] );
CodeGen codegen = vm->codegen();
codegen->vmiFUNCTION( makeName( "new", obj_K[ CLASS_OFFSET_TITLE ]), n, 1 );
//vmiCHECK_COUNT( codegen, n );
codegen->vmiSYS_CALL_ARGDAT( sysargdatConstruct, kk, n );
codegen->vmiSYS_RETURN();
vm->fastPeek() = codegen->vmiENDFUNCTION();
return pc;
}
void run(){
Lexer* lexer;
Perser* perser;
if(debug){
std::cout<<"Debug option\n";
}
lexer = new Lexer(debug);
lexer->load(input_file);
if(debug){
lexer->put_result();
}
perser = new Perser(lexer->getTokens(), debug);
if(!perser->perse()){
std::cout<<"Perse Error!!\n";
RELEASE(lexer);
RELEASE(perser);
return;
}
TranslationUnitAST* ast = perser->getAST();
CodeGen *codeGen = new CodeGen(debug);
if(!codeGen->codeGen( ast, input_file)){
std::cout<<"CodeGen Error!!\n";
RELEASE(lexer);
RELEASE(perser);
return;
}
llvm::Module &module = codeGen->getModule();
if(module.empty()){
std::cout<<" Module is empty!!\n";
RELEASE(lexer);
RELEASE(perser);
return;
}
llvm::PassManager pm;
pm.add(llvm::createPromoteMemoryToRegisterPass());
std::error_code error;
llvm::StringRef out_filename("do.out");
llvm::raw_fd_ostream raw_stream( out_filename, error, llvm::sys::fs::OpenFlags::F_None );
pm.add(createPrintModulePass( raw_stream));
pm.run(module);
raw_stream.close();
std::cout<<"Complate!!!!\n";
RELEASE(lexer);
RELEASE(perser);
}
void TableScanTranslator::ScanConsumer::FilterRowsByVisibility(
CodeGen &codegen, llvm::Value *tid_start, llvm::Value *tid_end,
Vector &selection_vector) const {
llvm::Value *executor_context_ptr =
translator_.GetCompilationContext().GetExecutorContextPtr();
llvm::Value *txn = codegen.Call(ExecutorContextProxy::GetTransaction,
{executor_context_ptr});
llvm::Value *raw_sel_vec = selection_vector.GetVectorPtr();
// Invoke TransactionRuntime::PerformRead(...)
llvm::Value *out_idx =
codegen.Call(TransactionRuntimeProxy::PerformVectorizedRead,
{txn, tile_group_ptr_, tid_start, tid_end, raw_sel_vec});
selection_vector.SetNumElements(out_idx);
}
/*
classExploder( key )( instance ) -> ( arg1, ..., argN )
*/
Ref * sysClassExploder( Ref * pc, MachineClass * vm ) {
if ( vm->count != 1 ) throw Ginger::Mishap( "Wrong number of arguments" );
Ref key = vm->fastPeek();
if ( !IsObj( key ) ) throw Ginger::Mishap( "Class of object needed" );
Ref * key_K = RefToPtr4( key );
if ( *key_K != sysClassKey ) throw Ginger::Mishap( "Class of object needed" );
const long N = SmallToLong( key_K[ CLASS_OFFSET_NFIELDS ] );
CodeGen codegen = vm->codegen();
codegen->vmiFUNCTION( makeName( "dest", key_K[ CLASS_OFFSET_TITLE ] ), 1, N );
codegen->vmiSYS_CALL_ARG( sysargExplode, key );
codegen->vmiSYS_RETURN();
vm->fastPeek() = codegen->vmiENDFUNCTION();
return pc;
}
codegen::Value Sorter::SorterAccess::LoadRowValue(
CodeGen &codegen, Sorter::SorterAccess::Row &row,
uint32_t column_index) const {
if (row.row_pos_ == nullptr) {
auto *addr = codegen->CreateInBoundsGEP(codegen.CharPtrType(), start_pos_,
row.row_idx_);
row.row_pos_ = codegen->CreateLoad(addr);
}
const auto &storage_format = sorter_.GetStorageFormat();
UpdateableStorage::NullBitmap null_bitmap(codegen, storage_format,
row.row_pos_);
return storage_format.GetValue(codegen, row.row_pos_, column_index,
null_bitmap);
}
Package* Compiler::Compile(const String& pkgName)
{
SSAssert(!m_compiled);
Package* ret = 0;
m_compiled = true;
g_compiler = this;
if(AnyErrors())
goto Failed;
m_pkg->SetName(pkgName);
{
StaticChecker checker(m_pkg);
if(!checker.Check())
goto Failed;
}
// CODE GENERATION
for(int i=0; i < m_pkg->GetClassCount(); ++i)
{
Class* cls = m_pkg->GetClass(i);
for(int j=0; j < cls->GetFunctionCount(); ++j)
{
Function* func = cls->GetFunction(j);
CodeGen gen;
gen.Generate(func);
}
}
if(AnyErrors())
goto Failed;
ret = m_pkg;
m_pkg = 0;
m_succeeded = true;
g_compiler = 0;
return ret;
Failed:
g_compiler = 0;
delete m_pkg;
m_pkg = 0;
return 0;
}
void Parser::VarListTail(ExprRec& var, ExprRec& index) {
switch (NextToken()) {
case COMMA:
Match(COMMA);
Variable(var, index);
code.ProcessVar(var, index); /*** CODE ***/
code.Listen(var, index); /*** CODE ***/
/* Recursion for other variables */
VarListTail(var, index);
break;
case SEMICOLON:
break;
default:
SyntaxError(NextToken(), "VarListTail");
}
}
/**
setSlot( CLASS:Class, POSITION:Small, METHOD:Method )
1. The method is inserted into the correct position in the class's
slot array. To start with, only one method per slot will be
permitted.
2. A call to setMethod is then made with an unsafe access function
as the method's function. The values are passed on the stack.
No stack checks are needed as the size of the argument lists of
the two functions are the same.
*/
Ref * sysSetSlot( Ref * pc, MachineClass * vm ) {
if ( vm->count != 3 ) throw Ginger::Mishap( "Wrong number of arguments" );
Ref method = vm->fastPop();
Ref position = vm->fastPop();
Ref gclass = vm->fastPop();
if ( !IsMethod( method ) ) throw Ginger::Mishap( "Method needed" ).culprit( "Method", refToString( method ) );
if ( !IsSmall( position ) ) throw Ginger::Mishap( "Small needed" ).culprit( "Position", refToString( position ) );
if ( !IsClass( gclass ) ) throw Ginger::Mishap( "Class needed" ).culprit( "Class", refToString( gclass ) );
long pos = SmallToLong( position );
long nfields = SmallToLong( RefToPtr4( gclass )[ CLASS_OFFSET_NFIELDS ] );
if ( not( 1 <= pos && pos <= nfields ) ) {
throw
Ginger::Mishap( "Position out of range" ).
culprit( "Position", pos ).
culprit( "Number of fields", nfields )
;
}
// Update the class-slot.
INDEX( INDEX( gclass, CLASS_OFFSET_SLOTS ), pos ) = method;
// Push onto the stack to get protection from garbage collection.
vm->fastPush( gclass );
vm->fastPush( method );
// ENDFUNCTION does not in fact cause a garbage collection, as it
// forces the heap to grow. However this is a more accurate way
// to write the code.
//
// The following block should not be in-lined but extracted as a
// service function.
{
CodeGen codegen = vm->codegen();
// TODO: Supply a useful name.
codegen->vmiFUNCTION( 1, 1 );
codegen->vmiFIELD( pos );
codegen->vmiSYS_RETURN();
vm->fastPush( codegen->vmiENDFUNCTION() );
}
// We do not need to modify vm->count, it's already 3.
// Simply chain into sysSetMethod.
return sysSetMethod( pc, vm );
}
void RowBatch::Row::SetValidity(CodeGen &codegen, llvm::Value *valid) {
if (valid->getType() != codegen.BoolType()) {
std::string error_msg;
llvm::raw_string_ostream rso{error_msg};
rso << "Validity of row must be a boolean value. Received type: "
<< valid->getType();
throw Exception{error_msg};
}
if (output_tracker_ == nullptr) {
throw Exception{"You didn't provide an output tracker for the row!"};
}
// Append this row to the output
llvm::Value *delta = codegen->CreateZExt(valid, codegen.Int32Type());
output_tracker_->AppendRowToOutput(codegen, *this, delta);
}
// Iterate over the tuples in the sorter in batches/vectors of the given size
void Sorter::VectorizedIterate(
CodeGen &codegen, llvm::Value *sorter_ptr, uint32_t vector_size,
uint64_t offset, Sorter::VectorizedIterateCallback &callback) const {
llvm::Value *start_pos = codegen.Load(SorterProxy::tuples_start, sorter_ptr);
llvm::Value *num_tuples = NumTuples(codegen, sorter_ptr);
num_tuples = codegen->CreateTrunc(num_tuples, codegen.Int32Type());
if (offset != 0) {
start_pos = codegen->CreateConstInBoundsGEP1_32(codegen.CharPtrType(),
start_pos, offset);
num_tuples = codegen->CreateSub(num_tuples, codegen.Const32(offset));
}
lang::VectorizedLoop loop(codegen, num_tuples, vector_size, {});
{
// Current loop range
auto curr_range = loop.GetCurrentRange();
// Provide an accessor into the sorted space
SorterAccess sorter_access(*this, start_pos);
// Issue the callback
callback.ProcessEntries(codegen, curr_range.start, curr_range.end,
sorter_access);
// That's it
loop.LoopEnd(codegen, {});
}
}
int main (int argc, char * argv[]) {
if (argc != 3) {
cout << "argument too few" << endl;
exit(1);
}
ifstream file(argv[1]);
if (file.fail ()) {
cout << "open file to read failed." << endl;
exit(1);
}
BufferReader reader(file);
Source source(reader);
MutCTokenizer * tokenizer = new MutCTokenizer(source);
MutCParser * parser = new MutCParser(tokenizer);
parser->parse ();
parser->ast ().printAST ();
cout << "start decorating..." << endl;
parser->decorateAST ();
cout << "decorating done..." << endl;
// shared_ptr<CodeGen> codegen = make_shared <CodeGen> (parser.get ());
CodeGen * codegen = new CodeGen(parser);
cout << "start generating..." << endl;
Code & code = codegen->generate ();
cout << "generating done..." << endl;
ofstream output(argv[2]);
if (output.fail ()) {
cout << "open file to write failed." << endl;
exit(1);
}
cout << "start outputing..." << endl;
output << code.toString ();
cout << "outputing done..." << endl;
delete codegen;
delete tokenizer;
return 0;
}
void Sorter::StoreTupleForTopK(CodeGen &codegen, llvm::Value *sorter_ptr,
const std::vector<codegen::Value> &tuple,
uint64_t top_k) const {
// Allocate room
auto *space = codegen.Call(SorterProxy::StoreTupleForTopK,
{sorter_ptr, codegen.Const64(top_k)});
// Serialize tuple
UpdateableStorage::NullBitmap null_bitmap(codegen, storage_format_, space);
for (uint32_t col_id = 0; col_id < tuple.size(); col_id++) {
storage_format_.SetValue(codegen, space, col_id, tuple[col_id],
null_bitmap);
}
null_bitmap.WriteBack(codegen);
// Finish
codegen.Call(SorterProxy::StoreTupleForTopKFinish,
{sorter_ptr, codegen.Const64(top_k)});
}
void Parser::ForStmt() {
/* Assignment */
Match(FOR_SYM);
Match(LBANANA);
ForAssign();
Match(SEMICOLON);
/* Condition */
ExprRec result;
code.ForTag(); /* TODO that is a new action symbol - add to grammar */
Condition(result);
code.ForBegin(result);
Match(SEMICOLON);
/* Increment variable */
ForAssign();
code.ForUpdate();
Match(RBANANA);
StmtList();
Match(END_SYM);
code.ForEnd();
}
codegen::Value Sorter::SorterAccess::LoadRowValue(
CodeGen &codegen, Sorter::SorterAccess::Row &row,
uint32_t column_index) const {
if (row.row_pos_ == nullptr) {
auto *tuple_size = sorter_.GetTupleSize(codegen);
auto *skip = codegen->CreateMul(row.row_idx_, tuple_size);
row.row_pos_ =
codegen->CreateInBoundsGEP(codegen.ByteType(), start_pos_, skip);
}
const auto &storage_format = sorter_.GetStorageFormat();
UpdateableStorage::NullBitmap null_bitmap{codegen, storage_format,
row.row_pos_};
if (!null_bitmap.IsNullable(column_index)) {
return storage_format.GetValueSkipNull(codegen, row.row_pos_, column_index);
} else {
return storage_format.GetValue(codegen, row.row_pos_, column_index,
null_bitmap);
}
}
Ref * sysClassUnsafeAccessor( Ref * pc, MachineClass *vm ) {
if ( vm->count != 2 ) throw Ginger::Mishap( "Wrong number of arguments" );
Ref N = vm->fastPop();
if ( !IsSmall( N ) ) throw Ginger::Mishap( "Integer index needed" );
Ref kk = vm->fastPeek();
if ( !isKey( kk ) ) throw Ginger::Mishap( "Key needed" );
long nargs = SmallToLong( RefToPtr4( kk )[ CLASS_OFFSET_NFIELDS ] );
long index = SmallToLong( N );
if ( 1 <= index && index <= nargs ) {
CodeGen codegen = vm->codegen();
// TODO: Figure out name.
codegen->vmiFUNCTION( 1, 1 );
codegen->vmiFIELD( index );
codegen->vmiSYS_RETURN();
vm->fastPeek() = codegen->vmiENDFUNCTION();
} else {
throw Ginger::Mishap( "ToBeDone" );
}
return pc;
}
