Esempio n. 1
0
int run(int buffsize, int partitionid, int totalpartitions)
{
	ScanHdf5Op node1;

	Config cfg;

	// init node1
	Setting& hdf5node = cfg.getRoot().add("hdf5scan", Setting::TypeGroup);
	hdf5node.add("file", Setting::TypeString) = hdf5filename;
	Setting& projattrnode = hdf5node.add("pick", Setting::TypeArray);
	projattrnode.add(Setting::TypeString) = "/candidate/id";
	projattrnode.add(Setting::TypeString) = "/candidate/sub_id";
	projattrnode.add(Setting::TypeString) = "/candidate/number";
	hdf5node.add("thispartition", Setting::TypeInt) = partitionid;
	hdf5node.add("totalpartitions", Setting::TypeInt) = totalpartitions;

	// init node2
	cfg.getRoot().add("path", Setting::TypeString) = "./";
	cfg.getRoot().add("buffsize", Setting::TypeInt) = buffsize;

	// build plan tree
	q.tree = &node1;

	// initialize each node
	node1.init(cfg, hdf5node);

	compute();

#ifdef VERBOSE
	cout << "---------- QUERY PLAN START ----------" << endl;
	PrettyPrinterVisitor ppv;
	q.accept(&ppv);
	cout << "----------- QUERY PLAN END -----------" << endl;
#endif

	q.destroynofree();

	return 0;
}
Esempio n. 2
0
int main()
{
	const int buffsize = 1 << 4;
	const int threads = THREADS;

	vector<string> tmpfileint;
	vector<string> tmpfiledouble;

	for (int i=0; i<threads; ++i)
	{
		ostringstream oss;
		oss << "testfileinttoint" << setfill('0') << setw(2) << i << ".tmp";
		tmpfileint.push_back(oss.str());
		createfileint(oss.str().c_str(), i, TUPLES);
	}
	for (int i=0; i<threads; ++i)
	{
		ostringstream oss;
		oss << "testfileinttodouble" << setfill('0') << setw(2) << i << ".tmp";
		tmpfiledouble.push_back(oss.str());
		createfiledouble(oss.str().c_str(), i, TUPLES);
	}

	Config cfg;

	cfg.getRoot().add("path", Setting::TypeString) = "./";
	cfg.getRoot().add("buffsize", Setting::TypeInt) = buffsize;

	// Init node1a
	Setting& scannode1 = cfg.getRoot().add("scan1", Setting::TypeGroup);
	scannode1.add("filetype", Setting::TypeString) = "text";
	Setting& files1 = scannode1.add("files", Setting::TypeList);
	Setting& mapping1 = scannode1.add("mapping", Setting::TypeList);
	for (int i=0; i<threads; ++i)
	{
		files1.add(Setting::TypeString) = tmpfileint.at(i);
		Setting& mapping1group = mapping1.add(Setting::TypeList);
		mapping1group.add(Setting::TypeInt) = i;
	}
	Setting& schemanode1 = scannode1.add("schema", Setting::TypeList);
	schemanode1.add(Setting::TypeString) = "long";
	schemanode1.add(Setting::TypeString) = "long";

	// Init node1b
	Setting& scannode2 = cfg.getRoot().add("scan2", Setting::TypeGroup);
	scannode2.add("filetype", Setting::TypeString) = "text";
	Setting& files2 = scannode2.add("files", Setting::TypeList);
	Setting& mapping2 = scannode2.add("mapping", Setting::TypeList);
	for (int i=0; i<threads; ++i)
	{
		files2.add(Setting::TypeString) = tmpfiledouble.at(i);
		Setting& mapping2group = mapping2.add(Setting::TypeList);
		mapping2group.add(Setting::TypeInt) = i;
	}
	Setting& schemanode2 = scannode2.add("schema", Setting::TypeList);
	schemanode2.add(Setting::TypeString) = "long";
	schemanode2.add(Setting::TypeString) = "dec";

	// Init node2
	Setting& joinnode = cfg.getRoot().add("join", Setting::TypeGroup);

	joinnode.add("mostfreqbuildkeyoccurances", Setting::TypeInt) = 1;

	// Partition group tree.
	Setting& pgnode = joinnode.add("threadgroups", Setting::TypeList);
	for (int i=0; i<threads; ++i)
	{
		Setting& singlepart = pgnode.add(Setting::TypeArray);
		singlepart.add(Setting::TypeInt) = i;
	}

	// Join attribute and projection tree.
	joinnode.add("buildjattr", Setting::TypeInt) = 0;
	joinnode.add("probejattr", Setting::TypeInt) = 0;

	Setting& projectnode = joinnode.add("projection", Setting::TypeList);
	projectnode.add(Setting::TypeString) = "B$1";
	projectnode.add(Setting::TypeString) = "P$1";

	// Init node3
	Setting& mergenode = cfg.getRoot().add("merge", Setting::TypeGroup);
	mergenode.add("threads", Setting::TypeInt) = threads;

//	cfg.write(stdout);

	// build plan tree
	q.tree = &node3;
	node3.nextOp = &node2;
	node2.buildOp = &node1a;
	node2.probeOp = &node1b;

	// initialize each node
	node1a.init(cfg, scannode1);
	node1b.init(cfg, scannode2);
	node2.init(cfg, joinnode);
	node3.init(cfg, mergenode);

#ifdef VERBOSE
	cout << "---------- QUERY PLAN START ----------" << endl;
	PrettyPrinterVisitor ppv;
	q.accept(&ppv);
	cout << "----------- QUERY PLAN END -----------" << endl;
#endif

	compute();

	for (int i=0; i<TUPLES*THREADS; ++i) 
	{
		const int correctcount = 1;
		if (verify[i] < correctcount)
			fail("Tuples are missing from output.");
		if (verify[i] > correctcount)
			fail("Extra tuples are in output.");
	}

	q.destroynofree();

	for (int i=0; i<threads; ++i)
	{
		deletefile(tmpfileint.at(i).c_str());
		deletefile(tmpfiledouble.at(i).c_str());
	}

	return 0;
}
int main()
{
	const int buffsize = 1 << 4;
	const int threads = 2;

	const char* tmpfileint1 = "testfileinttoint1.tmp";
	const char* tmpfileint2 = "testfileinttoint2.tmp";
	const char* tmpfiledouble = "testfileinttodouble.tmp";
	const char* tmpfileempty = "empty.tmp";

	Config cfg;

	createfileodd(tmpfileint1, TUPLES);
	createfileeven(tmpfileint2, TUPLES);
	createfilespecialdouble(tmpfiledouble);
	createfileeven(tmpfileempty, 0);

	cfg.getRoot().add("path", Setting::TypeString) = "./";
	cfg.getRoot().add("buffsize", Setting::TypeInt) = buffsize;

	// Init node1a
	Setting& scannode1 = cfg.getRoot().add("scan1", Setting::TypeGroup);
	scannode1.add("filetype", Setting::TypeString) = "text";
	Setting& files1 = scannode1.add("files", Setting::TypeList);
	files1.add(Setting::TypeString) = tmpfileint1;
	files1.add(Setting::TypeString) = tmpfileint2;
	Setting& mapping1 = scannode1.add("mapping", Setting::TypeList);
	Setting& mapping1group0 = mapping1.add(Setting::TypeList);
	mapping1group0.add(Setting::TypeInt) = 0;
	Setting& mapping1group1 = mapping1.add(Setting::TypeList);
	mapping1group1.add(Setting::TypeInt) = 1;
	Setting& schemanode1 = scannode1.add("schema", Setting::TypeList);
	schemanode1.add(Setting::TypeString) = "long";
	schemanode1.add(Setting::TypeString) = "long";

	// Init node1b
	Setting& scannode2 = cfg.getRoot().add("scan2", Setting::TypeGroup);
	scannode2.add("filetype", Setting::TypeString) = "text";
	Setting& files2 = scannode2.add("files", Setting::TypeList);
	files2.add(Setting::TypeString) = tmpfiledouble;
	files2.add(Setting::TypeString) = tmpfileempty;
	Setting& mapping2 = scannode2.add("mapping", Setting::TypeList);
	Setting& mapping2group0 = mapping2.add(Setting::TypeList);
	mapping2group0.add(Setting::TypeInt) = 0;
	Setting& mapping2group1 = mapping2.add(Setting::TypeList);
	mapping2group1.add(Setting::TypeInt) = 1;
	Setting& schemanode2 = scannode2.add("schema", Setting::TypeList);
	schemanode2.add(Setting::TypeString) = "long";
	schemanode2.add(Setting::TypeString) = "dec";

	// Init node2
	Setting& joinnode = cfg.getRoot().add("join", Setting::TypeGroup);

	joinnode.add("maxbuildtuples", Setting::TypeInt) = TUPLES * 2;
	joinnode.add("maxprobetuples", Setting::TypeInt) = TUPLES * 2;

	// Partition group tree.
	Setting& pgnode = joinnode.add("threadgroups", Setting::TypeList);
	Setting& partone = pgnode.add(Setting::TypeArray);
	for (int i=0; i<threads; ++i)
		partone.add(Setting::TypeInt) = i;

	// Join attribute and projection tree.
	joinnode.add("buildjattr", Setting::TypeInt) = 0;
	joinnode.add("probejattr", Setting::TypeInt) = 0;

	Setting& projectnode = joinnode.add("projection", Setting::TypeList);
	projectnode.add(Setting::TypeString) = "B$1";
	projectnode.add(Setting::TypeString) = "P$1";

	// Init node3
	Setting& mergenode = cfg.getRoot().add("merge", Setting::TypeGroup);
	mergenode.add("threads", Setting::TypeInt) = threads;

//	cfg.write(stdout);

	// build plan tree
	q.tree = &node3;
	node3.nextOp = &node2;
	node2.buildOp = &node1a;
	node2.probeOp = &node1b;

	// initialize each node
	node1a.init(cfg, scannode1);
	node1b.init(cfg, scannode2);
	node2.init(cfg, joinnode);
	node3.init(cfg, mergenode);

#ifdef VERBOSE
	cout << "---------- QUERY PLAN START ----------" << endl;
	PrettyPrinterVisitor ppv;
	q.accept(&ppv);
	cout << "----------- QUERY PLAN END -----------" << endl;
#endif

	compute();

	// Step 1: All ``vals'' keys present exactly once.
	// Erase verify array for next step.
	//
	for (unsigned int i=0; i<vals.size(); ++i) {
		if (verify.at(vals.at(i)) < 1)
			fail("Tuples are missing from output.");
		if (verify.at(vals.at(i)) > 1)
			fail("Expected value encountered multiple times.");
		verify.at(vals.at(i)) = 0;
	}

	// Setp 2: Verification array is empty, aka. nothing extra produced.
	//
	for (unsigned int i=0; i<verify.size(); ++i) {
		if (verify.at(i) != 0)
			fail("Extra tuples are in output.");
	}

	q.destroynofree();

	deletefile(tmpfileint1);
	deletefile(tmpfileint2);
	deletefile(tmpfiledouble);
	deletefile(tmpfileempty);

	return 0;
}
Esempio n. 4
0
int main()
{
	const int aggbuckets = 4;
	const int buffsize = 16;

	createaggfile(tempfilename, TUPLES);

	AggregateSum node1;
	MergeOp mergeop;
	AggregateSum node2;
	ParallelScanOp node3;

	Config cfg;

	// init node1
	Setting& aggnode1 = cfg.getRoot().add("aggsumpre", Setting::TypeGroup);
	aggnode1.add("field", Setting::TypeInt) = 0;
	aggnode1.add("sumfield", Setting::TypeInt) = 1;
	Setting& agghashnode1 = aggnode1.add("hash", Setting::TypeGroup);
	agghashnode1.add("fn", Setting::TypeString) = "modulo";
	agghashnode1.add("buckets", Setting::TypeInt) = aggbuckets;
	agghashnode1.add("field", Setting::TypeInt) = 0;
	
	// init mergeop
	Setting& mergenode = cfg.getRoot().add("merge", Setting::TypeGroup);
	mergenode.add("threads", Setting::TypeInt) = THREADS;

	// init node2
	Setting& aggnode2 = cfg.getRoot().add("aggsumpost", Setting::TypeGroup);
	aggnode2.add("field", Setting::TypeInt) = 0;
	aggnode2.add("sumfield", Setting::TypeInt) = 1;
	Setting& agghashnode2 = aggnode2.add("hash", Setting::TypeGroup);
	agghashnode2.add("fn", Setting::TypeString) = "modulo";
	agghashnode2.add("buckets", Setting::TypeInt) = aggbuckets;
	agghashnode2.add("field", Setting::TypeInt) = 0;
	
	// init node3
	cfg.getRoot().add("path", Setting::TypeString) = "./";
	cfg.getRoot().add("buffsize", Setting::TypeInt) = buffsize;

	Setting& scannode = cfg.getRoot().add("scan", Setting::TypeGroup);
	scannode.add("filetype", Setting::TypeString) = "text";
	Setting& files = scannode.add("files", Setting::TypeList);
	files.add(Setting::TypeString) = tempfilename;
	Setting& mapping = scannode.add("mapping", Setting::TypeList);
	Setting& mappinggroup0 = mapping.add(Setting::TypeList);
	for (int i=0; i<THREADS; ++i)
		mappinggroup0.add(Setting::TypeInt) = i;
	Setting& schemanode = scannode.add("schema", Setting::TypeList);
	schemanode.add(Setting::TypeString) = "long";
	schemanode.add(Setting::TypeString) = "dec";

	// build plan tree
	q.tree = &node1;
	node1.nextOp = &mergeop;
	mergeop.nextOp = &node2;
	node2.nextOp = &node3;

	// initialize each node
	node3.init(cfg, scannode);
	node2.init(cfg, aggnode2);
	mergeop.init(cfg, mergenode);
	node1.init(cfg, aggnode1);

#ifdef VERBOSE
	cout << "---------- QUERY PLAN START ----------" << endl;
	PrettyPrinterVisitor ppv;
	q.accept(&ppv);
	cout << "----------- QUERY PLAN END -----------" << endl;
#endif

	compute();

	q.destroynofree();

	deletefile(tempfilename);

	return 0;
}
Esempio n. 5
0
int main()
{
    const int buffsize = 1 << 4;
    const int threads = 4;

    const char* tmpfileint = "testfileinttoint.tmp";
    const char* tmpfiledouble = "testfileinttodouble.tmp";

    Config cfg;

    createfile(tmpfileint, TUPLES);
    createfiledouble(tmpfiledouble, TUPLES);

    cfg.getRoot().add("path", Setting::TypeString) = "./";
    cfg.getRoot().add("buffsize", Setting::TypeInt) = buffsize;

    // Init node1a
    Setting& scannode1 = cfg.getRoot().add("scan1", Setting::TypeGroup);
    scannode1.add("filetype", Setting::TypeString) = "text";
    Setting& files1 = scannode1.add("files", Setting::TypeList);
    files1.add(Setting::TypeString) = tmpfileint;
    Setting& mapping1 = scannode1.add("mapping", Setting::TypeList);
    Setting& mapping1group0 = mapping1.add(Setting::TypeList);
    for (int i=0; i<threads; ++i)
        mapping1group0.add(Setting::TypeInt) = i;
    Setting& schemanode1 = scannode1.add("schema", Setting::TypeList);
    schemanode1.add(Setting::TypeString) = "long";
    schemanode1.add(Setting::TypeString) = "long";

    // Init node1b
    Setting& scannode2 = cfg.getRoot().add("scan2", Setting::TypeGroup);
    scannode2.add("filetype", Setting::TypeString) = "text";
    Setting& files2 = scannode2.add("files", Setting::TypeList);
    files2.add(Setting::TypeString) = tmpfiledouble;
    Setting& mapping2 = scannode2.add("mapping", Setting::TypeList);
    Setting& mapping2group0 = mapping2.add(Setting::TypeList);
    for (int i=0; i<threads; ++i)
        mapping2group0.add(Setting::TypeInt) = i;
    Setting& schemanode2 = scannode2.add("schema", Setting::TypeList);
    schemanode2.add(Setting::TypeString) = "long";
    schemanode2.add(Setting::TypeString) = "dec";

    // Init node2
    Setting& joinnode = cfg.getRoot().add("join", Setting::TypeGroup);

    joinnode.add("maxbuildtuples", Setting::TypeInt) = TUPLES * 2;
    joinnode.add("maxprobetuples", Setting::TypeInt) = TUPLES * 2;

    // Partition group tree.
    Setting& pgnode = joinnode.add("threadgroups", Setting::TypeList);
    Setting& singlepart = pgnode.add(Setting::TypeArray);
    for (int i=0; i<threads; ++i)
        singlepart.add(Setting::TypeInt) = i;

    // Join attribute and projection tree.
    joinnode.add("buildjattr", Setting::TypeInt) = 0;
    joinnode.add("probejattr", Setting::TypeInt) = 0;

    Setting& projectnode = joinnode.add("projection", Setting::TypeList);
    projectnode.add(Setting::TypeString) = "B$1";
    projectnode.add(Setting::TypeString) = "P$1";

    // Init node3
    Setting& mergenode = cfg.getRoot().add("merge", Setting::TypeGroup);
    mergenode.add("threads", Setting::TypeInt) = threads;

//	cfg.write(stdout);

    // build plan tree
    q.tree = &node3;
    node3.nextOp = &node2;
    node2.buildOp = &node1a;
    node2.probeOp = &node1b;

    // initialize each node
    node1a.init(cfg, scannode1);
    node1b.init(cfg, scannode2);
    node2.init(cfg, joinnode);
    node3.init(cfg, mergenode);

#ifdef VERBOSE
    cout << "---------- QUERY PLAN START ----------" << endl;
    PrettyPrinterVisitor ppv;
    q.accept(&ppv);
    cout << "----------- QUERY PLAN END -----------" << endl;
#endif

    compute();

    for (int i=0; i<TUPLES; ++i) {
        if (verify[i] < 1)
            fail("Tuples are missing from output.");
        if (verify[i] > 1)
            fail("Extra tuples are in output.");
    }

    q.destroynofree();

    deletefile(tmpfileint);
    deletefile(tmpfiledouble);

    return 0;
}