int main(int argc, char *argv[]) {
int child = fork();
if (child == 0) {
ExtentTypeLibrary library;
const ExtentType::Ptr type = library.registerTypePtr(type_string);
DataSeriesSink sink("incomplete-ds-file.ds", Extent::compression_algs[Extent::compress_mode_lzf].compress_flag);
ExtentSeries series(type);
OutputModule output(sink, series, type, 4 * 1024);
Int64Field number(series, "number");
sink.writeExtentLibrary(library);
for (uint64_t i = 0; i < 64 * 1024; i++) {
output.newRecord();
number.set(i);
if (i == 50 * 1024) {
output.flushExtent();
sink.flushPending();
}
assert(i < 50 * 1024);
}
}
int error = waitpid(child, NULL, 0);
if (error != child) {
return -1;
}
return 0;
}
int main(int argc,char *argv[]) {
commonPackingArgs packing_args;
getPackingArgs(&argc,argv,&packing_args);
INVARIANT(argc == 3,
format("Usage: %s [ds-common-args] inname outdsname; - valid for inname")
% argv[0]);
FILE *infile;
if (strcmp(argv[1],"-")==0) {
infile = stdin;
} else {
infile = fopen(argv[1],"r");
INVARIANT(infile != NULL, format("Unable to open file %s: %s")
% argv[1] % strerror(errno));
}
DataSeriesSink outds(argv[2],
packing_args.compress_modes,
packing_args.compress_level);
ExtentTypeLibrary library;
const ExtentType::Ptr extent_type(library.registerTypePtr(web_wc_custom_xml));
series.setType(extent_type);
outmodule = new OutputModule(outds, series, extent_type, packing_args.extent_size);
outds.writeExtentLibrary(library);
record buf;
while (true) {
size_t amt = fread(&buf, 1, sizeof(record), infile);
if (amt == 0) {
SINVARIANT(feof(infile) && ! ferror(infile));
break;
}
SINVARIANT(amt == sizeof(record));
++nrecords;
outmodule->newRecord();
timestamp.set(ntohl(buf.timestamp));
clientID.set(ntohl(buf.clientID));
objectID.set(ntohl(buf.objectID));
size.set(ntohl(buf.size));
method.set(buf.method);
status.set(buf.status);
type.set(buf.type);
server.set(buf.server);
}
cout << format("Processed %d records\n") % nrecords;
delete outmodule;
outds.close();
return 0;
}
void doAnimation(const vector<string> &args, const commonPackingArgs &packing_args) {
NFSDSAnalysisMod::registerUnitsEpoch();
TypeIndexModule *sourcea = new TypeIndexModule("NFS trace: common");
sourcea->setSecondMatch("Trace::NFS::common");
TypeIndexModule *sourceb = new TypeIndexModule("NFS trace: attr-ops");
sourceb->setSecondMatch("Trace::NFS::attr-ops");
TypeIndexModule *sourcec = new TypeIndexModule("NFS trace: read-write");
sourcec->setSecondMatch("Trace::NFS::read-write");
for (unsigned i = 1; i < args.size() - 1; ++i) {
sourcea->addSource(args[i]);
}
sourceb->sameInputFiles(*sourcea);
sourcec->sameInputFiles(*sourcea);
SequenceModule seq_common(sourcea);
SequenceModule seq_attr(sourceb);
NFSDSModule *attr_common_join = NFSDSAnalysisMod::newAttrOpsCommonJoin();
NFSDSAnalysisMod::setAttrOpsSources(attr_common_join, seq_common, seq_attr);
SequenceModule seq_common_attr(attr_common_join);
SequenceModule seq_rw(sourcec);
NFSDSModule *attr_common_rw_join = NFSDSAnalysisMod::newCommonAttrRWJoin();
NFSDSAnalysisMod::setCommonAttrRWSources(attr_common_rw_join, seq_common_attr, seq_rw);
if (false) {
DStoTextModule ds_to_text(*attr_common_rw_join);
ds_to_text.getAndDeleteShared();
return;
}
DataSeriesSink outds(args.back(), packing_args.compress_modes, packing_args.compress_level);
ExtentTypeLibrary library;
const ExtentType::Ptr extent_type(library.registerTypePtr(output_xml));
ExtentSeries out_series(extent_type);
OutputModule out_module(outds, out_series, extent_type, packing_args.extent_size);
outds.writeExtentLibrary(library);
AnimationConvert convert(*attr_common_rw_join, out_module);
convert.getAndDeleteShared();
}
void test_empty_field_name() {
MersenneTwisterRandom rand;
cout << format("test_empty_field_name - start seed=%d\n") % rand.seed_used;
ExtentTypeLibrary typelib;
typelib.registerTypePtr("<ExtentType name=\"Test::EmptyFieldName\" >\n"
" <field type=\"int32\" name=\"int32a\" />\n"
" <field type=\"int32\" name=\"int32b\" />\n"
"</ExtentType>\n");
ExtentSeries series1(typelib,"Test::EmptyFieldName");
series1.newExtent();
int nrecords = 1000 + rand.randInt(10000);
series1.createRecords(nrecords);
Int32Field f_int32a(series1, "int32a");
Int32Field f_int32b(series1, "int32b");
for (int i=0; i < nrecords; ++i) {
f_int32a.set(rand.randInt());
f_int32b.set(rand.randInt());
series1.next();
SINVARIANT(series1.more());
}
series1.setExtent(series1.getSharedExtent());
Int32Field f_switch(series1, "");
int counta = 0, countb = 0;
for (int i=0; i < nrecords; ++i) {
SINVARIANT(series1.more());
bool which = rand.randInt(2) == 0;
if (which) {
++counta;
if (f_switch.getName() != "int32a" || rand.randInt(2) == 0) {
f_switch.setFieldName("int32a");
}
SINVARIANT(f_switch.val() == f_int32a.val());
} else {
++countb;
if (f_switch.getName() != "int32b" || rand.randInt(2) == 0) {
f_switch.setFieldName("int32b");
}
SINVARIANT(f_switch.val() == f_int32b.val());
}
}
SINVARIANT(counta > nrecords / 3 && countb > nrecords / 3);
cout << "passed empty_field_name tests\n";
}
const ExtentType::Ptr getXMLDescFromFile(const string &filename, ExtentTypeLibrary &lib) {
ifstream xml_desc_input(filename.c_str());
INVARIANT(xml_desc_input.good(),
format("error opening %s: %s") % filename % strerror(errno));
string xml_desc;
while (!xml_desc_input.eof()) {
char buf[8192];
buf[0] = '\0';
xml_desc_input.read(buf, 8192);
INVARIANT(xml_desc_input.good() || xml_desc_input.eof(),
format("error reading %s: %s") % filename % strerror(errno));
streamsize read_bytes = xml_desc_input.gcount();
INVARIANT(read_bytes > 0 || xml_desc_input.eof(), format("error reading %s: %s")
% filename % strerror(errno));
xml_desc.append(string(buf, read_bytes));
}
LintelLogDebug("csv2ds::xml", format("XML description:\n%s") % xml_desc);
return lib.registerTypePtr(xml_desc);
}
void
test_doublebase_nullable()
{
printf("test_doublebase_nullable...\n");
string dbntype_xml(
"<ExtentType name=\"doublebase nullable test type\">\n"
" <field type=\"double\" name=\"double\" opt_nullable=\"yes\" opt_doublebase=\"1000000\" pack_scale=\"1000\" />\n"
"</ExtentType>");
ExtentTypeLibrary library;
const ExtentType::Ptr dbntype(library.registerTypePtr(dbntype_xml));
ExtentSeries dbnseries(dbntype);
DoubleField f_double(dbnseries,"double",
Field::flag_nullable | DoubleField::flag_allownonzerobase);
Extent::Ptr cur_extent(new Extent(dbntype));
dbnseries.setExtent(cur_extent);
dbnseries.newRecord();
f_double.setNull();
SINVARIANT(f_double.isNull());
dbnseries.newRecord();
f_double.set(1000000);
SINVARIANT(!f_double.isNull());
dbnseries.newRecord();
f_double.setabs(1000000);
SINVARIANT(!f_double.isNull());
dbnseries.setExtent(cur_extent);
SINVARIANT(dbnseries.morerecords());
SINVARIANT(f_double.isNull());
SINVARIANT(0 == f_double.val());
SINVARIANT(1000000 == f_double.absval()); // changed semantics to have offset in absval always
++dbnseries;
SINVARIANT(dbnseries.morerecords());
SINVARIANT(false == f_double.isNull());
SINVARIANT(1000000 == f_double.val());
SINVARIANT(2000000 == f_double.absval());
++dbnseries;
SINVARIANT(dbnseries.morerecords());
SINVARIANT(false == f_double.isNull());
SINVARIANT(0 == f_double.val());
SINVARIANT(1000000 == f_double.absval());
}
int main(int argc, char *argv[]) {
ExtentTypeLibrary library;
const ExtentType::Ptr type = library.registerTypePtr(type_string);
DataSeriesSink sink("extent-write-callback.ds", Extent::compression_algs[Extent::compress_mode_lzf].compress_flag);
ExtentSeries series(type);
OutputModule output(sink, series, type, 4 * 1024);
Int64Field number(series, "number");
sink.setExtentWriteCallback(&printIndex);
sink.writeExtentLibrary(library);
for (uint64_t i = 0; i < 64 * 1024; i++) {
output.newRecord();
number.set(i);
}
}
void doEllard(const vector<string> &args, const commonPackingArgs &packing_args) {
NFSDSAnalysisMod::registerUnitsEpoch();
TypeIndexModule *source = new TypeIndexModule("Trace::NFS::Ellard");
for (unsigned i = 1; i < args.size() - 1; ++i) {
source->addSource(args[i]);
}
DataSeriesSink outds(args.back(), packing_args.compress_modes, packing_args.compress_level);
ExtentTypeLibrary library;
const ExtentType::Ptr extent_type(library.registerTypePtr(output_xml));
ExtentSeries out_series(extent_type);
OutputModule out_module(outds, out_series, extent_type, packing_args.extent_size);
outds.writeExtentLibrary(library);
EllardConvert convert(*source, out_module);
convert.getAndDeleteShared();
convert.printResult();
}
int main(int argc, char *argv[]) {
Extent::setReadChecksFromEnv(true); // going to be compressing, may as well check
commonPackingArgs packing_args;
getPackingArgs(&argc,argv,&packing_args);
if (string(argv[1]) == "--update-type") {
update_type = true;
for (int i = 2; i < argc; i++) {
argv[i-1] = argv[i];
}
argc--;
}
lintel::programOptionsHelp("[common-args] [options] type-prefix field,field,...\n"
" input-filenames... output-filename\n"
"common-args include:\n");
lintel::programOptionsHelp(packingOptions());
vector<string> extra_args = lintel::parseCommandLine(argc, argv, true);
if (extra_args.size() < 4) {
lintel::programOptionsUsage(argv[0]);
exit(0);
}
string type_prefix(extra_args[0]);
string fieldlist(extra_args[1]);
vector<string> fields;
split(fieldlist,",",fields);
if (fieldlist == "all") {
get_all_fields = true;
fields.pop_back();
}
{
struct stat buf;
INVARIANT(stat(extra_args.back().c_str(), &buf) != 0,
format("Refusing to overwrite existing file %s.")
% extra_args.back());
}
DataSeriesSink output(extra_args.back(), packing_args.compress_modes,
packing_args.compress_level);
// to get the complete typename and type information...
DataSeriesSource first_file(extra_args[2]);
const ExtentType::Ptr intype = first_file.getLibrary().getTypeByPrefixPtr(type_prefix);
first_file.closefile();
INVARIANT(intype != NULL, boost::format("can not find a type matching prefix %s")
% type_prefix);
TypeIndexModule source(intype->getName());
for (unsigned i=2; i < (extra_args.size()-1); ++i) {
source.addSource(extra_args[i]);
}
source.startPrefetching();
ExtentSeries inputseries(ExtentSeries::typeLoose);
ExtentSeries outputseries(ExtentSeries::typeLoose);
vector<GeneralField *> infields, outfields;
// TODO: figure out how to handle pack_relative options that are
// specified relative to a field that was not selected. Right
// now, there is no way to do this selection; try replacing
// enter_driver with leave_driver in
// src/Makefile.am:ran.check-dsselect. Also, give an option for the
// xml description to be specified on the command line.
string xmloutdesc(str(format("<ExtentType name=\"%s\" namespace=\"%s\" version=\"%d.%d\">\n")
% intype->getName() % intype->getNamespace() % intype->majorVersion()
% intype->minorVersion()));
inputseries.setType(intype);
if (get_all_fields) {
for (uint i=0; i < intype->getNFields(); i++) {
fields.push_back( intype->getFieldName(i) );
}
}
for (vector<string>::iterator i = fields.begin();
i != fields.end();++i) {
cout << format("%s -> %s\n") % *i % intype->xmlFieldDesc(*i);
xmloutdesc.append(str(format(" %s\n") % intype->xmlFieldDesc(*i)));
infields.push_back(GeneralField::create(NULL,inputseries,*i));
}
xmloutdesc.append("</ExtentType>\n");
cout << xmloutdesc << "\n";
if (update_type) {
cout << "Update this type ? (y/n)";
char c;
cin >> c;
if (c == 'y') {
ofstream otypeinfo("temp.txt");
otypeinfo << xmloutdesc;
otypeinfo.close();
cout << "Xml info has been written to temp.txt, hit enter to continue when done editing" << endl;
cin.clear();
cin.ignore(1,'/n');
ifstream itypeinfo("temp.txt");
xmloutdesc.assign( (istreambuf_iterator<char>(itypeinfo)) , (istreambuf_iterator<char>()) );
itypeinfo.close();
remove("temp.txt");
}
}
ExtentTypeLibrary library;
const ExtentType::Ptr outputtype(library.registerTypePtr(xmloutdesc));
output.writeExtentLibrary(library);
outputseries.setType(outputtype);
for (vector<string>::iterator i = fields.begin();
i != fields.end();++i) {
outfields.push_back(GeneralField::create(NULL,outputseries,*i));
}
DSExpr *where = NULL;
if (where_arg.used()) {
string tmp = where_arg.get();
where = DSExpr::make(inputseries, tmp);
}
OutputModule outmodule(output,outputseries,outputtype,
packing_args.extent_size);
uint64_t input_row_count = 0, output_row_count = 0;
while (true) {
Extent::Ptr inextent = source.getSharedExtent();
if (inextent == NULL)
break;
for (inputseries.setExtent(inextent);inputseries.morerecords(); ++inputseries) {
++input_row_count;
if (where && !where->valBool()) {
continue;
}
++output_row_count;
outmodule.newRecord();
for (unsigned int i=0;i<infields.size();++i) {
outfields[i]->set(infields[i]);
}
}
}
outmodule.flushExtent();
outmodule.close();
GeneralField::deleteFields(infields);
GeneralField::deleteFields(outfields);
delete where;
cout << format("%d input rows, %d output rows\n") % input_row_count % output_row_count;
return 0;
}
void
test_compactnull()
{
MersenneTwisterRandom rand;
cout << format("test_compactnull - start seed=%d\n") % rand.seed_used;
ExtentTypeLibrary typelib;
// One real bool, 7 hidden ones
typelib.registerTypePtr
("<ExtentType name=\"Test::CompactNulls\" pack_null_compact=\"non_bool\" >\n"
" <field type=\"bool\" name=\"bool\" opt_nullable=\"yes\" />\n"
" <field type=\"byte\" name=\"byte\" opt_nullable=\"yes\" />\n"
" <field type=\"int32\" name=\"int32\" opt_nullable=\"yes\" pack_relative=\"int32\" />\n"
" <field type=\"int32\" name=\"int32b\" opt_nullable=\"yes\" pack_relative=\"int32\" />\n"
" <field type=\"int64\" name=\"int64\" opt_nullable=\"yes\" pack_relative=\"int64\" />\n"
" <field type=\"double\" name=\"double\" opt_nullable=\"yes\" />\n"
" <field type=\"variable32\" name=\"variable32\" opt_nullable=\"yes\" pack_unique=\"yes\" />\n"
"</ExtentType>\n");
ExtentSeries series1(typelib,"Test::CompactNulls");
series1.newExtent();
int nrecords = 1000 + rand.randInt(10000);
series1.createRecords(nrecords);
BoolField f_bool(series1, "bool", Field::flag_nullable);
ByteField f_byte(series1, "byte", Field::flag_nullable);
Int32Field f_int32(series1, "int32", Field::flag_nullable);
Int32Field f_int32b(series1, "int32b", Field::flag_nullable);
Int64Field f_int64(series1, "int64", Field::flag_nullable);
DoubleField f_double(series1, "double", Field::flag_nullable);
Variable32Field f_variable32(series1, "variable32", Field::flag_nullable);
Extent::ByteArray variablestuff;
variablestuff.resize(nrecords * 4 + 5);
for (int i=0;i<nrecords * 2 + 5;i++) {
variablestuff[i] = (char)(i&0xFF);
}
/// Test 1: all null
// Test filling in a value and then nulling.
for (int i=1;i<=nrecords;i++) {
f_bool.set(true);
f_bool.setNull();
f_byte.set(i & 0xFF);
f_byte.setNull();
f_int32.set(i);
f_int32.setNull();
f_int32b.set(i*10);
f_int32b.setNull();
f_int64.set(i*7731);
f_int64.setNull();
f_double.set(i+10000);
f_double.setNull();
f_variable32.set(variablestuff.begin()+i,i+1);
f_variable32.setNull();
++series1;
}
Extent::ByteArray packed;
series1.getExtentRef().packData(packed,
Extent::compression_algs[Extent::compress_mode_none].compress_flag);
cout << format("all null: %d rows, original bytes %d, packed %d\n")
% nrecords % series1.getExtentRef().size() % packed.size();
uint32_t overhead = 48 + (4 - (nrecords % 4)) % 4;
INVARIANT(packed.size() == static_cast<size_t>(overhead + nrecords),
"size check failed");
ExtentSeries series2(typelib, "Test::CompactNulls");
series2.newExtent();
series2.getExtentRef().unpackData(packed, false);
series1.setExtent(series1.getSharedExtent());
for (int i=0;i<nrecords;i++) {
INVARIANT(f_bool.isNull() && f_byte.isNull() && f_int32.isNull()
&& f_int32b.isNull() && f_int64.isNull()
&& f_double.isNull() && f_variable32.isNull(), "??");
++series1;
}
/// Test 2: random nulls
cout << "all null uncompact passed\n";
series1.setExtent(series1.getSharedExtent());
// Fill at random...
for (int i=1;i<=nrecords;i++) {
f_bool.set(true);
if (rand.randInt(2)) f_bool.setNull();
f_byte.set(i & 0xFF);
if (rand.randInt(2)) f_byte.setNull();
f_int32.set(i);
if (rand.randInt(2)) f_int32.setNull();
f_int32b.set(rand.randInt());
if (rand.randInt(2)) f_int32b.setNull();
f_int64.set(i*7731);
if (rand.randInt(2)) f_int64.setNull();
f_double.set(i+10000);
if (rand.randInt(2)) f_double.setNull();
f_variable32.set(variablestuff.begin()+i,i+1);
if (rand.randInt(2)) f_variable32.setNull();
++series1;
}
packed.clear();
series1.getExtentRef().packData(packed,
Extent::compression_algs[Extent::compress_mode_lzf].compress_flag);
cout << format("random null: %d rows, original bytes %d, packed %d\n")
% nrecords % series1.getExtentRef().size() % packed.size();
BoolField g_bool(series2, "", Field::flag_nullable);
ByteField g_byte(series2, "byte", Field::flag_nullable);
Int32Field g_int32(series2, "int32", Field::flag_nullable);
Int32Field g_int32b(series2, "int32b", Field::flag_nullable);
Int64Field g_int64(series2, "int64", Field::flag_nullable);
DoubleField g_double(series2, "double", Field::flag_nullable);
Variable32Field g_variable32(series2, "variable32", Field::flag_nullable);
series2.getExtentRef().unpackData(packed, false);
series1.setExtent(series1.getSharedExtent());
series2.setExtent(series2.getSharedExtent());
for (int i=0;i<nrecords;i++) {
g_bool.setFieldName("bool"); // slow and inefficient, but for testing
INVARIANT((f_bool.isNull() && g_bool.isNull())
|| (!f_bool.isNull() && !g_bool.isNull() &&
f_bool.val() == g_bool.val()),
format("bad@%d %d %d/%d %d")
% i % f_bool.isNull() % g_bool.isNull()
% f_bool.val() % g_bool.val());
SINVARIANT((f_byte.isNull() && g_byte.isNull())
|| (!f_byte.isNull() && !g_byte.isNull() &&
f_byte.val() == g_byte.val()));
SINVARIANT((f_int32.isNull() && g_int32.isNull())
|| (!f_int32.isNull() && !g_int32.isNull() &&
f_int32.val() == g_int32.val()));
SINVARIANT((f_int32b.isNull() && g_int32b.isNull())
|| (!f_int32b.isNull() && !g_int32b.isNull() &&
f_int32b.val() == g_int32b.val()));
SINVARIANT((f_int64.isNull() && g_int64.isNull())
|| (!f_int64.isNull() && !g_int64.isNull() &&
f_int64.val() == g_int64.val()));
SINVARIANT((f_double.isNull() && g_double.isNull())
|| (!f_double.isNull() && !g_double.isNull() &&
f_double.val() == g_double.val()));
SINVARIANT((f_variable32.isNull() && g_variable32.isNull())
|| (!f_variable32.isNull() && !g_variable32.isNull() &&
f_variable32.stringval() == g_variable32.stringval()));
++series1;
++series2;
}
cout << "random null passed\n";
cout << "test_compactnull - end\n";
}
void
test_makecomplexfile()
{
printf("test_makecomplexfile...\n");
string complextype(
"<ExtentType name=\"complex test type\">\n"
" <field type=\"bool\" name=\"bool\" />\n"
" <field type=\"bool\" name=\"null_bool\" opt_nullable=\"yes\"/>\n"
" <field type=\"byte\" name=\"byte\" />\n"
" <field type=\"byte\" name=\"null_byte\" opt_nullable=\"yes\"/>\n"
" <field type=\"int32\" name=\"int32\" />\n"
" <field type=\"int32\" name=\"null_int32\" opt_nullable=\"yes\"/>\n"
" <field type=\"int64\" name=\"int64\" />\n"
" <field type=\"int64\" name=\"null_int64\" opt_nullable=\"yes\"/>\n"
" <field type=\"double\" name=\"double\" />\n"
" <field type=\"double\" name=\"null_double\" opt_nullable=\"yes\"/>\n"
" <field type=\"double\" name=\"base_double\" opt_doublebase=\"100000\"/>\n"
" <field type=\"variable32\" name=\"variable32\" />\n"
" <field type=\"variable32\" name=\"null_variable32\" opt_nullable=\"yes\"/>\n"
"</ExtentType>");
ExtentTypeLibrary library;
const ExtentType::Ptr outputtype(library.registerTypePtr(complextype));
DataSeriesSink output("misc.ds");
output.writeExtentLibrary(library);
ExtentSeries outputseries(outputtype);
OutputModule outmodule(output,outputseries,outputtype,30000);
BoolField f_bool(outputseries,"bool");
BoolField f_bool_null(outputseries,"null_bool",Field::flag_nullable);
ByteField f_byte(outputseries,"byte");
ByteField f_byte_null(outputseries,"null_byte",Field::flag_nullable);
Int32Field f_int32(outputseries,"int32");
Int32Field f_int32_null(outputseries,"null_int32",Field::flag_nullable);
Int64Field f_int64(outputseries,"int64");
Int64Field f_int64_null(outputseries,"null_int64",Field::flag_nullable);
DoubleField f_double(outputseries,"double");
DoubleField f_double_null(outputseries,"null_double",Field::flag_nullable);
DoubleField f_double_base(outputseries,"base_double",DoubleField::flag_allownonzerobase);
Variable32Field f_variable32(outputseries,"variable32");
Variable32Field f_variable32_null(outputseries,"null_variable32",Field::flag_nullable);
MersenneTwisterRandom rand(1781);
for (int i=0;i<10000;++i) {
outmodule.newRecord();
f_bool.set(randBool(rand));
if (randBool(rand)) {
f_bool_null.setNull(true);
} else {
f_bool_null.set(randBool(rand));
SINVARIANT(f_bool_null.isNull() == false);
}
f_byte.set((ExtentType::byte)(rand.randInt() & 0xFF));
if (randBool(rand)) {
f_byte_null.setNull(true);
} else {
f_byte_null.set((ExtentType::byte)(rand.randInt() & 0xFF));
}
f_int32.set(rand.randInt());
if (randBool(rand)) {
f_int32_null.setNull(true);
} else {
f_int32_null.set(rand.randInt());
}
f_int64.set(rand.randLongLong());
if (randBool(rand)) {
f_int64_null.setNull(true);
} else {
f_int64_null.set(rand.randLongLong());
}
f_double.set(rand.randDoubleOpen53());
if (randBool(rand)) {
f_double_null.setNull(true);
} else {
f_double_null.set(rand.randDoubleOpen53());
}
f_double_base.setabs(rand.randDoubleOpen53() * 1000000.0);
const int vdsize = 32;
char variabledata[vdsize];
fillVariableData(rand,variabledata,vdsize);
f_variable32.set(variabledata,vdsize);
if (randBool(rand)) {
f_variable32_null.setNull(true);
} else {
fillVariableData(rand,variabledata,vdsize);
f_variable32_null.set(variabledata,vdsize);
}
}
outmodule.flushExtent();
printf("test_makecomplexfile - Done\n");
}
void
test_extentpackunpack()
{
printf("test_extentpackunpack - start\n");
ExtentTypeLibrary typelib;
typelib.registerTypePtr
("<ExtentType name=\"test type\">\n"
" <field type=\"int32\" name=\"input1\" pack_relative=\"input1\" />\n"
" <field type=\"int32\" name=\"input2\" pack_relative=\"input1\" />\n"
" <field type=\"int64\" name=\"int64-1\" pack_relative=\"int64-1\" />\n"
" <field type=\"int64\" name=\"int64-2\" pack_relative=\"int64-2\" />\n"
" <field type=\"double\" name=\"double1\" pack_scale=\"1e-6\" pack_relative=\"double1\" />\n"
" <field type=\"variable32\" name=\"var1\"/>\n"
" <field type=\"variable32\" name=\"var2\"/>\n"
"</ExtentType>\n");
ExtentSeries testseries(typelib,"test type");
testseries.newExtent();
const int nrecords = 937;
testseries.createRecords(nrecords);
Int32Field int1(testseries,"input1");
Int32Field int2(testseries,"input2");
Int64Field int64_1(testseries,"int64-1");
Int64Field int64_2(testseries,"int64-2");
DoubleField double1(testseries,"double1");
Variable32Field var1(testseries,"var1");
Variable32Field var2(testseries,"var2");
Extent::ByteArray variablestuff;
variablestuff.resize(nrecords * 2 + 5);
for (int i=0;i<nrecords * 2 + 5;i++) {
variablestuff[i] = (char)(i&0xFF);
}
for (int i=0;i<nrecords;i++) {
int1.set(i);
int2.set(nrecords-i);
int64_1.set((ExtentType::int64)i * (ExtentType::int64)1000000 * (ExtentType::int64)1000000);
int64_2.set((ExtentType::int64)i * (ExtentType::int64)19721776 * (ExtentType::int64)1000000);
double1.set((double)i/1000000.0);
SINVARIANT(var1.size() == 0);
var1.set(variablestuff.begin(),i+1);
SINVARIANT(var1.size() == i+1);
SINVARIANT(memcmp(var1.val(),variablestuff.begin(),i+1)==0);
SINVARIANT(var2.size() == 0);
var2.set(variablestuff.begin(),2*i+1);
SINVARIANT(var2.size() == 2*i+1);
SINVARIANT(memcmp(var2.val(),variablestuff.begin(),2*i+1)==0);
++testseries;
}
testseries.setExtent(testseries.getSharedExtent());
for (int i=0;i<nrecords;i++) {
INVARIANT(int1.val() == i, format("?? %d") % int1.val());
SINVARIANT(int2.val() == nrecords-i);;
SINVARIANT(int64_1.val() == (ExtentType::int64)i * (ExtentType::int64)1000000 * (ExtentType::int64)1000000);
SINVARIANT(int64_2.val() == (ExtentType::int64)i * (ExtentType::int64)19721776 * (ExtentType::int64)1000000);
SINVARIANT((int)round(double1.val() * 1000000.0) == i);
SINVARIANT(var1.size() == i+1);
SINVARIANT(memcmp(var1.val(),variablestuff.begin(),i+1)==0);
SINVARIANT(var2.size() == 2*i+1);
SINVARIANT(memcmp(var2.val(),variablestuff.begin(),2*i+1)==0);
++testseries;
}
Extent::ByteArray packed;
testseries.getExtentRef().packData(packed,
Extent::compression_algs[Extent::compress_mode_zlib].compress_flag);
Extent::Ptr unpackextent(new Extent(testseries.getTypePtr()));
unpackextent->unpackData(packed, false);
testseries.setExtent(unpackextent);
for (int i=0;i<nrecords;i++) {
INVARIANT(int1.val() == i, format("?? %d %d") % int1.val() % i);
SINVARIANT(int2.val() == nrecords-i);;
SINVARIANT(int64_1.val() == (ExtentType::int64)i * (ExtentType::int64)1000000 * (ExtentType::int64)1000000);
SINVARIANT(int64_2.val() == (ExtentType::int64)i * (ExtentType::int64)19721776 * (ExtentType::int64)1000000);
SINVARIANT((int)round(double1.val() * 1000000.0) == i);
SINVARIANT(var1.size() == i+1);
SINVARIANT(memcmp(var1.val(),variablestuff.begin(),i+1)==0);
SINVARIANT(var2.size() == 2*i+1);
SINVARIANT(memcmp(var2.val(),variablestuff.begin(),2*i+1)==0);
++testseries;
}
cout << format("unpacked bytes %d, packed %d\n")
% unpackextent->size() % packed.size();
printf("test_extentpackunpack - end\n");
}
void checkRegisterUnitsEpoch() {
boost::mt19937 rng;
uint32_t seed =
lintel::BobJenkinsHashMix3(getpid(), getppid(),
lintel::BobJenkinsHashMixULL(Clock::todTfrac()));
if (getenv("SEED") != NULL) {
seed = stringToInteger<uint32_t>(getenv("SEED"));
}
rng.seed(seed);
cout << format("register units epoch seed %d\n") % seed;
ExtentTypeLibrary lib;
const ExtentType::Ptr type(lib.registerTypePtr
("<ExtentType name=\"test\" namespace=\"test\" version=\"1.0\" >\n"
" <field type=\"int64\" name=\"nsec_unix\" />"
" <field type=\"int64\" name=\"usec_unix\" />"
" <field type=\"int64\" name=\"frac32_unix\" />"
" <field type=\"int64\" name=\"override\" units=\"nanoseconds\" epoch=\"unix\" />"
"</ExtentType>\n"));
ExtentSeries s(type);
s.newExtent();
Int64TimeField::registerUnitsEpoch("nsec_unix", "test", "test", 1, "nanoseconds", "unix");
Int64TimeField::registerUnitsEpoch("usec_unix", "test", "test", 1, "microseconds", "unix");
Int64TimeField::registerUnitsEpoch("frac32_unix", "test", "test", 1, "2^-32 seconds", "unix");
Int64TimeField::registerUnitsEpoch("override", "test", "test", 1, "microseconds", "unix");
Int64TimeField nsec_unix(s, "nsec_unix");
Int64TimeField usec_unix(s, "usec_unix");
Int64TimeField frac32_unix(s, "frac32_unix");
Int64TimeField override(s, "override"); // will be microseconds
s.createRecords(1);
for (uint32_t i = 0; i < 10; ++i) {
double seconds = (rng() % (1000*1000)) + (rng() % 1000000)/1.0e6;
int64_t nsec = static_cast<int64_t>(round(seconds * 1.0e9));
int64_t usec = static_cast<int64_t>(round(seconds * 1.0e6));
double frac32_d = seconds * (4*1024.0*1024.0*1024.0);
int64_t frac32 = static_cast<int64_t>(round(frac32_d));
nsec_unix.setRaw(nsec);
usec_unix.setRaw(usec);
frac32_unix.setRaw(frac32);
override.setRaw(usec);
SINVARIANT(nsec_unix.valFrac32() == usec_unix.valFrac32());
// SEED=1357899212;
// 951475.894468 -> 4086557849672407.500, rounding happens differently
// here than in Int64TimeField.
INVARIANT(fabs(usec_unix.valFrac32() - frac32_d) <= 0.55,
format("%.6f -> %.3f - %d = %.2f") % seconds % frac32_d
% usec_unix.valFrac32() % (frac32_d - usec_unix.valFrac32()));
SINVARIANT(frac32_unix.valFrac32() == frac32);
SINVARIANT(usec_unix.valFrac32() == override.valFrac32());
SINVARIANT(nsec_unix.valSecNano() == usec_unix.valSecNano());
SINVARIANT(usec_unix.valSecNano() == frac32_unix.valSecNano());
SINVARIANT(frac32_unix.valSecNano() == override.valSecNano());
}
input_series.setExtent(in);
string output_xml(str(format("<ExtentType name=\"expr-transform -> %s\""
" namespace=\"server.example.com\" version=\"1.0\">\n")
% output_table_name));
BOOST_FOREACH(ExprColumn &column, expr_columns) {
string extra;
if (column.type == "variable32") {
extra = "pack_unique=\"yes\"";
}
output_xml.append(str(format(" <field type=\"%s\" name=\"%s\" opt_nullable=\"yes\""
" %s/>\n") % column.type % column.name % extra));
}
output_xml.append("</ExtentType>\n");
ExtentTypeLibrary lib;
output_series.setType(lib.registerTypePtr(output_xml));
previous_row_series.setType(output_series.getTypePtr());
previous_row_series.newExtent();
previous_row_series.newRecord();
copier.prep();
DSExprParser::FieldNameToSelector field_name_to_selector
= boost::bind(&ExprTransformModule::fieldNameToSeries, this, _1);
boost::scoped_ptr<DSExprParser> parser(DSExprParser::MakeDefaultParser());
outputs.reserve(expr_columns.size());
for (size_t i = 0; i < expr_columns.size(); ++i) {
const ExprColumn &column(expr_columns[i]);
boost::shared_ptr<DSExpr> expr(parser->parse(field_name_to_selector, column.expr));
GeneralField::Ptr field = GeneralField::make(output_series, column.name);
int main(int argc, char *argv[]) {
if (argc != 7) {
std::cout << "Wrong usage!\n";
std::cout << "Usage: ds-simple-write <timestamp> <filename> "
"<opcode> <offset> <iosize> <outfile>\n";
std::cout << "Field types:\n";
std::cout << "\ttimestamp - int64\n";
std::cout << "\tfilename - variable32 (string)\n";
std::cout << "\topcode - byte (\"r\" or \"w\")\n";
std::cout << "\toffset - int64\n";
std::cout << "\tiosize - int64\n";
return 1;
}
const char *outfile = argv[6];
/*
* First, let'a define a schema for our extent. Don't forget to
* include namespace and version information: you will get a warning if
* you don't have either a namespace or a version. Remember to follow
* the heirarchical naming conventions when naming extents. It's mostly
* like Trace::<type of trace>::<Machine type>::
* <workload type>::<anything else that is appropriate>
*/
const char *extentXMLDescription =
"<ExtentType namespace=\"http://www.fsl.cs.sunysb.edu/\""
" name=\"Trace::Fictitious::Linux\""
" version=\"0.1\""
" comment=\"an example tracing type\">\n"
" <field type=\"int64\" name=\"timestamp\""
" comment=\"a helpful comment explaining the meaning of the field.\"/>\n"
" <field type=\"variable32\" name=\"filename\"/>\n"
" <field type=\"byte\" name=\"opcode\" comment=\"0=read 1=write 2=other\" />\n"
" <field type=\"int64\" name=\"offset\"/>\n"
" <field type=\"int64\" name=\"iosize\"/>\n"
"</ExtentType>\n";
ExtentTypeLibrary extentTypeLibrary;
const ExtentType::Ptr extentType =
extentTypeLibrary.registerTypePtr(extentXMLDescription);
/*
* Sink is a wrapper for a DataSeries output file.
* Create a sink and write out the extent type extent.
*/
DataSeriesSink outfileSink(outfile);
outfileSink.writeExtentLibrary(extentTypeLibrary);
/*
* Now create an output module that
* processes extents using ExtentSeries (iterator)
* and put them into the sink.
*/
uint32_t target_extent_size = 4096;
// Note you would normally want larger extents, e.g. 96k
// Since only one row will be generated, it doesn't matter here.
ExtentSeries extentSeries;
OutputModule outputModule(outfileSink, extentSeries,
extentType, target_extent_size);
/*
* These are handles for the fields in the
* "current record" of extentSeries.
*/
Int64Field timestamp(extentSeries, "timestamp");
Variable32Field filename(extentSeries, "filename");
ByteField opcode(extentSeries, "opcode");
Int64Field offset(extentSeries, "offset");
Int64Field size(extentSeries, "iosize");
/*
* Initiate a new record in extentSeries
*/
outputModule.newRecord();
/*
* Set new records of the extentSeries
*/
timestamp.set(stringToInteger<int64_t>(argv[1]));
filename.set(argv[2]);
if (!strcmp(argv[3], "r"))
opcode.set((ExtentType::byte)0x0);
else if (!strcmp(argv[3], "w"))
opcode.set((ExtentType::byte)0x1);
else
opcode.set((ExtentType::byte)0x2);
offset.set(stringToInteger<int64_t>(argv[4]));
size.set(stringToInteger<int64_t>(argv[5]));
/*
* Ask output module to finilize the file
*/
outputModule.flushExtent();
outputModule.close();
return 0;
}
