bool TestRunner::truncateTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "truncateTest" );
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "a", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for writing";
logMsg( ERROR, errstr.str() );
return false;
}
// choose a random block to truncate at
struct drand48_data d48data;
srand48_r(0xdeadbeef, &d48data);
long int blk_num;
lrand48_r( &d48data, &blk_num);
blk_num = blk_num % m_opts.numblocks;
// always leave at least one block
if( blk_num == 0 )
blk_num = 1;
int rc = m_file->truncate(blk_num * BLK_SIZE);
if( (filetype != IDBDataFile::HDFS) && rc)
{
logMsg( ERROR, "truncate failed!" );
return false;
}
else if ( (filetype == IDBDataFile::HDFS) && !rc )
{
logMsg( ERROR, "truncate is supposed to fail for HDFS files!" );
return false;
}
else if( (filetype == IDBDataFile::HDFS) )
{
// this is the "success" case for HDFS we didn't expect to truncate so reset blk_num
blk_num = m_opts.numblocks;
}
off64_t fsize = m_file->size();
if( fsize != (off64_t) (blk_num * BLK_SIZE))
{
ostringstream errstr;
errstr << "wrong file size after truncate, " << fsize << " != " << blk_num*BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::hdfsRdwrExhaustTest()
{
// this is going to be a self-contained test that attempts to test
// all logic inherent in HdfsRdwr
// choose a new filename that is specific to our thread
ostringstream oss;
// embed pid so that this is a new directory path
oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id;
string newpath = oss.str();
// open a file with arbitrarily small buffer
IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
assert( file );
// check various empty file conditions
assert( file->size() == 0 );
assert( file->tell() == 0 );
assert( file->seek(-1, SEEK_CUR) == -1);
assert( file->seek(0, SEEK_SET) == 0);
unsigned char buf[4];
assert( file->read(buf, 4) == 0);
// write some data
buf[0] = 0xde; buf[1] = 0xad; buf[2] = 0xbe; buf[3] = 0xef;
assert( file->write(buf, 4) == 4);
assert( file->size() == 4 );
assert( file->tell() == 4 );
assert( file->truncate(-1) == -1 );
// now make file empty again
assert( file->truncate(0) == 0 );
assert( file->size() == 0 );
assert( file->seek(0, SEEK_SET) == 0);
assert( file->tell() == 0 );
assert( file->read(buf, 4) == 0);
// write data again, this time exactly up to allocated size
assert( file->write(buf, 4) == 4);
assert( file->write(buf, 4) == 4);
assert( file->size() == 8 );
assert( file->tell() == 8 );
// truncate back to 4
assert( file->truncate(4) == 0 );
assert( file->size() == 4 );
assert( file->seek(4, SEEK_SET) == 0);
assert( file->tell() == 4 );
// now trigger a buffer reallocation
assert( file->write(buf, 4) == 4);
assert( file->write(buf, 4) == 4);
assert( file->size() == 12 );
// now delete and close.
delete file;
// check the file size through the file system
IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS );
assert( fs.size( newpath.c_str() ) == 12);
// open again - the file is bigger than the default buffer so it triggers alternate
// logic in the constructor
file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
assert( file );
assert( file->size() == 12);
unsigned char newbuf[4];
assert( file->pread(newbuf, 4, 4) == 4);
assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe &&newbuf[3] == 0xef);
delete file;
fs.remove(newpath.c_str());
return true;
}
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
m_open_opts = open_opts;
ostringstream infostr;
infostr << "Running test for file type "
<< ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
logMsg( INFO, infostr.str() );
IDBFileSystem& fs = IDBFileSystem::getFs( filetype );
// build the file name we are going to use
ostringstream oss;
// embed pid so that this is a new directory path
oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id;
string dir = oss.str();
m_fname = dir + "/foobar";
// instantiate this here so that we always clean up files we created no matter
// where we exit the function from
boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );
logMsg( INFO, "Running writeTest" );
bool returnval = true;
returnval = writeTest( filetype );
if( !returnval )
{
return false;
}
// going to check the size two different ways - once through the file and
// once through the file system.
unsigned fsize = m_file->size();
if( fsize != m_opts.numBlocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from file " << fsize << " != " << m_opts.numBlocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
// this deletes and closes the file - required to get accurate file size with
// buffered IO and in HDFS from the file system
reset();
fsize = fs.size(m_fname.c_str());
if( fsize != m_opts.numBlocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from fs " << fsize << " != " << m_opts.numBlocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
if( returnval )
{
logMsg( INFO, "Running readTest" );
returnval = readTest( filetype );
}
if( returnval )
{
logMsg( INFO, "Running rdwrTest" );
returnval = rdwrTest( filetype );
}
if( returnval && filetype == IDBDataFile::HDFS )
{
logMsg( INFO, "Running hdfsRdwrExhaustTest" );
returnval = hdfsRdwrExhaustTest();
}
if( m_opts.numDbRoots > 0 )
{
logMsg( INFO, "Checking dbroots" );
for( int i = 0; i < m_opts.numDbRoots; ++i )
{
ostringstream dbroot;
dbroot << "/usr/local/Calpont/data" << i+1;
if( !fs.exists(dbroot.str().c_str()) )
{
ostringstream msg;
msg << "Could not locate dbroot directory " << dbroot.str();
logMsg( ERROR, msg.str() );
returnval = false;
}
}
}
list<string> dircontents;
assert( fs.listDirectory( dir.c_str(), dircontents ) == 0 );
ostringstream ldstr;
ldstr << "Listed directory " << dir << ":";
list<string>::iterator iend = dircontents.end();
for( list<string>::iterator i = dircontents.begin(); i != iend; ++i )
{
ldstr << (*i) << ",";
}
logMsg( INFO, ldstr.str() );
assert( dircontents.size() == 1 );
// now check a bogus path and make sure it returns -1
assert( fs.listDirectory( "/this-is-a-bogus-directory", dircontents ) == -1 );
assert( fs.remove( "/this-is-a-bogus-directory" ) == 0 );
assert( !fs.isDir( "/this-is-a-bogus-directory" ));
assert( !fs.isDir( m_fname.c_str() ));
if( returnval )
{
logMsg( INFO, "All tests passed!" );
}
reset();
return returnval;
}
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
m_open_opts = open_opts;
ostringstream infostr;
string typeString = ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
infostr << "Running test for file type " << typeString;
logMsg( INFO, infostr.str() );
if (filetype != IDBDataFile::UNBUFFERED && filetype != IDBDataFile::BUFFERED)
{
if( !IDBPolicy::installPlugin(m_opts.pluginFile) )
{
cout << "ERROR: unable to install HDFS plugin!" << endl;
return -1;
}
}
IDBFileSystem& fs = IDBFileSystem::getFs( filetype );
// build the file name we are going to use
ostringstream oss;
// embed pid so that this is a new directory path
oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id << "/Calpont/data";
// we need to make sure this directory doesn't already exist
// todo-this only works non-HDFS
string cmd = "rm -rf " + oss.str();
system(cmd.c_str());
string dir = oss.str();
m_fname = dir + "/foobar";
// instantiate this here so that we always clean up files we created no matter
// where we exit the function from
boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );
// HDFS will automatically create parent directories when opening a file so these
// tests around mkdir are irrelevant.
if( filetype != IDBDataFile::HDFS )
{
// we expect this to fail because the parent directory should not exist
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts);
if( m_file )
{
ostringstream errstr;
errstr << "open for writing of path " << m_fname << " should not succeed!";
logMsg( ERROR, errstr.str() );
return false;
}
// now create the path
if (fs.mkdir(dir.c_str()))
{
ostringstream errstr;
errstr << "open mkdir of " << dir << " failed!";
logMsg( ERROR, errstr.str() );
return false;
}
}
bool returnval = true;
if( returnval )
{
returnval = writeTest( filetype );
}
// going to check the size two different ways - once through the file and
// once through the file system.
unsigned fsize = m_file->size();
if( fsize != m_opts.numblocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from file " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
// this deletes and closes the file - required to get accurate file size with
// buffered IO and in HDFS from the file system
reset();
fsize = fs.size(m_fname.c_str());
if( fsize != m_opts.numblocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from fs " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
if( returnval )
{
returnval = tellTest( filetype );
}
if( returnval )
{
returnval = readTest( filetype );
}
if( returnval && (filetype == IDBDataFile::UNBUFFERED || filetype == IDBDataFile::HDFS ))
{
returnval = concurrencyTest( filetype );
}
if( returnval )
{
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for reading";
logMsg( ERROR, errstr.str() );
return false;
}
if( m_opts.numthreads > 1)
{
boost::thread_group thread_group;
for( int i = 0; i < m_opts.numthreads; ++i )
{
thread_group.create_thread(boost::bind(thread_func2, boost::ref(*this)));
}
thread_group.join_all();
}
returnval = randomReadTest();
}
if( returnval )
{
returnval = rdwrTest( filetype );
}
if( returnval && filetype == IDBDataFile::BUFFERED )
{
returnval = openByModeStrTest();
}
if( returnval )
{
returnval = truncateTest( filetype );
}
if( returnval )
{
returnval = renameTest( filetype );
}
if( returnval )
{
returnval = copyTest( filetype );
}
if( returnval && filetype == IDBDataFile::HDFS )
{
returnval = hdfsRdwrExhaustTest();
}
if( returnval )
{
returnval = flushTest( filetype );
}
if( returnval )
{
returnval = seekTest( filetype );
}
if( returnval )
{
returnval = listDirTest( filetype, dir );
}
if( returnval )
logMsg( INFO, typeString + " tests passed!", true );
reset();
return returnval;
}