bool TestRunner::tellTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "tellTest" );
reset();
// scenario: reader opens file, seeks somewhere and tells where it is.
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
assert(m_file);
// read a few blocks
assert( readBlock(0, m_defbuf, 0) );
assert( readBlock(1, m_defbuf, 1) );
if( m_file->seek(BLK_SIZE, SEEK_SET) )
{
ostringstream errstr;
errstr << "tellTest: failed to seek block";
logMsg( ERROR, errstr.str() );
return false;
}
off64_t filepos = m_file->tell();
if( filepos != off64_t(BLK_SIZE) )
{
ostringstream errstr;
errstr << "tellTest: File position not at correct block, " << filepos << " != " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::openByModeStrTest()
{
logMsg( INFO, "openByModeStrTest" );
// in this test we want to check the alternate open modes available for buffered i/o
// this test is only run if we are doing buffered I/O and expects it is run after the
// write test to guarantee the file is there
reset();
m_file = IDBDataFile::open(IDBDataFile::BUFFERED, m_fname.c_str(), "r+b", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for read/write";
logMsg( ERROR, errstr.str() );
return false;
}
// keep this fairly simple - read a block then write a block
ssize_t readct = m_file->read(m_defbuf,BLK_SIZE);
if( (size_t) readct != BLK_SIZE )
{
ostringstream errstr;
errstr << "Only read " << readct << " bytes, expected 4";
logMsg( ERROR, errstr.str() );
return false;
}
if( m_defbuf[0] != (unsigned char) 0 )
{
ostringstream errstr;
errstr << "Data error - expected " << 0 << ", read " << (int) m_defbuf[0];
logMsg( ERROR, errstr.str() );
return false;
}
// we should be at block 1
long filepos = m_file->tell();
if( filepos != long(BLK_SIZE) )
{
ostringstream errstr;
errstr << "File position not at correct block, " << filepos << " != " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
m_defbuf[0] = 1;
ssize_t bytes_written = m_file->write(m_defbuf, BLK_SIZE);
if( (size_t) bytes_written != BLK_SIZE )
{
ostringstream errstr;
errstr << "Only wrote " << bytes_written << " bytes, expected 4";
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::truncateTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "truncateTest" );
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "a", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for writing";
logMsg( ERROR, errstr.str() );
return false;
}
// choose a random block to truncate at
struct drand48_data d48data;
srand48_r(0xdeadbeef, &d48data);
long int blk_num;
lrand48_r( &d48data, &blk_num);
blk_num = blk_num % m_opts.numblocks;
// always leave at least one block
if( blk_num == 0 )
blk_num = 1;
int rc = m_file->truncate(blk_num * BLK_SIZE);
if( (filetype != IDBDataFile::HDFS) && rc)
{
logMsg( ERROR, "truncate failed!" );
return false;
}
else if ( (filetype == IDBDataFile::HDFS) && !rc )
{
logMsg( ERROR, "truncate is supposed to fail for HDFS files!" );
return false;
}
else if( (filetype == IDBDataFile::HDFS) )
{
// this is the "success" case for HDFS we didn't expect to truncate so reset blk_num
blk_num = m_opts.numblocks;
}
off64_t fsize = m_file->size();
if( fsize != (off64_t) (blk_num * BLK_SIZE))
{
ostringstream errstr;
errstr << "wrong file size after truncate, " << fsize << " != " << blk_num*BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::concurrencyTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "concurrencyTest" );
reset();
// ok - scenario is a reader opens the file then a reader/writer opens
// the same file and updates something, then the reader tries to read
// again.
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
assert(m_file);
// read a few blocks
assert( readBlock(0, m_defbuf, 0) );
assert( readBlock(1, m_defbuf, 1) );
// open the same file for read/write
IDBDataFile* rdwrFile = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
assert(rdwrFile);
// read a block, write a block
assert( rdwrFile->pread(m_defbuf, 0, BLK_SIZE) == (ssize_t) BLK_SIZE );
assert( m_defbuf[0] == 0 );
m_defbuf[0] = 95;
assert( rdwrFile->seek(0, 0) == 0 );
assert( rdwrFile->write(m_defbuf, BLK_SIZE) );
// close file
delete rdwrFile;
// this 5 seconds is important in the HDFS case because it gives HDFS
// time to age off (or whatever) the blocks for the original file that
// have now been rewritten. The value was determined experimentally -
// 3 secs works fine most times but not all. If HDFS hasn't aged out
// the blocks then the read will return the old data
if( filetype == IDBDataFile::HDFS )
sleep(10);
// go back to the reader and make sure he got the new value, then close
assert( readBlock(0, m_defbuf, 95) );
delete m_file;
// now put block 0 back the way it was
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
assert(m_file);
assert( writeBlock(0, m_defbuf, 0) );
return true;
}
bool TestRunner::doBlock(unsigned int blocknum, unsigned char tag, unsigned int count)
{
writeBlocks(blocknum,m_defbuf,tag,count);
m_file->flush();
if( m_opts.closeonwrite )
{
delete m_file;
// will have to cache type somewhere later
m_file = IDBDataFile::open( IDBDataFile::HDFS, m_fname.c_str(), "a", m_open_opts );
if( !m_file )
return false;
}
if( m_opts.reopen )
{
delete m_file;
// will have to cache type somewhere later
m_file = IDBDataFile::open( IDBDataFile::HDFS, m_fname.c_str(), "r", m_open_opts );
if( !m_file )
return false;
}
unsigned char buf[BLK_SIZE];
return readBlocks(blocknum,buf,tag,count);
}
bool TestRunner::writeBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag)
{
buf[0] = tag;
size_t rc = m_file->write(buf, BLK_SIZE);
if (rc != BLK_SIZE)
{
ostringstream errstr;
errstr << "writeBlock failed for block " << blocknum << ", wrote " << rc << " bytes, expecting " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::randomReadTest()
{
logMsg( INFO, "randomReadTest" );
struct drand48_data d48data;
srand48_r(pthread_self(), &d48data);
for( int i = 0; i < 10; ++i )
{
long int blk_num;
lrand48_r( &d48data, &blk_num);
blk_num = blk_num % m_opts.numblocks;
unsigned char readbuf[4];
assert( m_file->pread( readbuf, blk_num * BLK_SIZE, 4 ) == 4 );
assert( readbuf[0] == (unsigned char) blk_num );
}
return true;
}
bool TestRunner::readTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "readTest" );
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for reading";
logMsg( ERROR, errstr.str() );
return false;
}
// check the mtime method.
time_t now = time(0);
time_t mtime = m_file->mtime();
assert( (now-mtime) <= 3 );
struct timeval starttime,endtime,timediff;
gettimeofday(&starttime,0x0);
for( int i = 0; i < m_opts.numblocks; ++i)
{
if( !readBlock(i, m_defbuf, i) )
return false;
}
gettimeofday(&endtime,0x0);
timeval_subtract(&timediff,&endtime,&starttime);
float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001);
ostringstream infostr;
infostr << "Read " << m_opts.numblocks * BLK_SIZE << " bytes in " << secs << " secs, ";
infostr << "Throughput = " << setprecision(3) << ((m_opts.numblocks * BLK_SIZE) / 1000000.0) / secs << "MB/sec";
logMsg( INFO, infostr.str() );
return true;
}
bool TestRunner::readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag)
{
size_t rc = m_file->pread(buf,blocknum*BLK_SIZE,BLK_SIZE);
//cout << "DEBUG: read " << rc << " bytes at offset " << blocknum * BLK_SIZE << endl;
if (rc != BLK_SIZE)
{
ostringstream errstr;
errstr << "readBlock failed for block " << blocknum << ", read " << rc << " bytes, expecting " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
else if (tag != buf[0])
{
ostringstream errstr;
errstr << "read tag 0x" << setw(2) << hex << setfill('0') << (int) buf[0] << " at block " << dec
<< blocknum << ", expected 0x" << (int) tag;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::hdfsRdwrExhaustTest()
{
// this is going to be a self-contained test that attempts to test
// all logic inherent in HdfsRdwr
// choose a new filename that is specific to our thread
ostringstream oss;
// embed pid so that this is a new directory path
oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id;
string newpath = oss.str();
// open a file with arbitrarily small buffer
IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
assert( file );
// check various empty file conditions
assert( file->size() == 0 );
assert( file->tell() == 0 );
assert( file->seek(-1, SEEK_CUR) == -1);
assert( file->seek(0, SEEK_SET) == 0);
unsigned char buf[4];
assert( file->read(buf, 4) == 0);
// write some data
buf[0] = 0xde; buf[1] = 0xad; buf[2] = 0xbe; buf[3] = 0xef;
assert( file->write(buf, 4) == 4);
assert( file->size() == 4 );
assert( file->tell() == 4 );
assert( file->truncate(-1) == -1 );
// now make file empty again
assert( file->truncate(0) == 0 );
assert( file->size() == 0 );
assert( file->seek(0, SEEK_SET) == 0);
assert( file->tell() == 0 );
assert( file->read(buf, 4) == 0);
// write data again, this time exactly up to allocated size
assert( file->write(buf, 4) == 4);
assert( file->write(buf, 4) == 4);
assert( file->size() == 8 );
assert( file->tell() == 8 );
// truncate back to 4
assert( file->truncate(4) == 0 );
assert( file->size() == 4 );
assert( file->seek(4, SEEK_SET) == 0);
assert( file->tell() == 4 );
// now trigger a buffer reallocation
assert( file->write(buf, 4) == 4);
assert( file->write(buf, 4) == 4);
assert( file->size() == 12 );
// now delete and close.
delete file;
// check the file size through the file system
IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS );
assert( fs.size( newpath.c_str() ) == 12);
// open again - the file is bigger than the default buffer so it triggers alternate
// logic in the constructor
file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
assert( file );
assert( file->size() == 12);
unsigned char newbuf[4];
assert( file->pread(newbuf, 4, 4) == 4);
assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe &&newbuf[3] == 0xef);
delete file;
fs.remove(newpath.c_str());
return true;
}
bool TestRunner::rdwrTest( IDBDataFile::Types filetype )
{
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for reading";
logMsg( ERROR, errstr.str() );
return false;
}
struct drand48_data d48data;
srand48_r(0xdeadbeef, &d48data);
// we will write to 5 random blocks and then come back and
// verify the contents and undo them
int blocks_to_touch = min( 5, m_opts.numBlocks);
vector<int> touched;
for( int i = 0; i < blocks_to_touch; ++i )
{
long int blk_num;
// we need to make sure all the blocks we touch are unique or
// the pattern used by this test won't work
bool found = false;
while(!found)
{
lrand48_r( &d48data, &blk_num);
blk_num = blk_num % m_opts.numBlocks;
vector<int>::iterator pos = find( touched.begin(), touched.end(), blk_num );
if( pos == touched.end())
found = true;
}
if( m_file->seek(blk_num * BLK_SIZE, SEEK_SET) )
{
ostringstream errstr;
errstr << "failed to seek block " << blk_num;
logMsg( ERROR, errstr.str() );
return false;
}
unsigned char writeval = 0xb0;
size_t writect = m_file->write(&writeval,1);
if( writect != 1 )
{
ostringstream errstr;
errstr << "Only wrote " << writect << " bytes, expected 1";
logMsg( ERROR, errstr.str() );
return false;
}
touched.push_back(blk_num);
}
m_file->flush();
for( int i = 0; i < (int) touched.size(); ++i )
{
unsigned char readbuf;
size_t readct = m_file->pread(&readbuf,touched[i] * BLK_SIZE,1);
if( readct != 1 || readbuf != (unsigned char) 0xb0 )
{
ostringstream errstr;
errstr << "Error reading expected value, readct=" << readct << " bytes, value" << (int) readbuf;
logMsg( ERROR, errstr.str() );
return false;
}
readbuf = touched[i];
if( m_file->seek(-1, SEEK_CUR) )
{
ostringstream errstr;
errstr << "failed to seek -1";
logMsg( ERROR, errstr.str() );
return false;
}
size_t writect = m_file->write(&readbuf,1);
if( writect != 1 )
{
ostringstream errstr;
errstr << "Only wrote " << writect << " bytes, expected 1";
logMsg( ERROR, errstr.str() );
return false;
}
}
return true;
}
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
m_open_opts = open_opts;
ostringstream infostr;
infostr << "Running test for file type "
<< ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
logMsg( INFO, infostr.str() );
IDBFileSystem& fs = IDBFileSystem::getFs( filetype );
// build the file name we are going to use
ostringstream oss;
// embed pid so that this is a new directory path
oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id;
string dir = oss.str();
m_fname = dir + "/foobar";
// instantiate this here so that we always clean up files we created no matter
// where we exit the function from
boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );
logMsg( INFO, "Running writeTest" );
bool returnval = true;
returnval = writeTest( filetype );
if( !returnval )
{
return false;
}
// going to check the size two different ways - once through the file and
// once through the file system.
unsigned fsize = m_file->size();
if( fsize != m_opts.numBlocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from file " << fsize << " != " << m_opts.numBlocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
// this deletes and closes the file - required to get accurate file size with
// buffered IO and in HDFS from the file system
reset();
fsize = fs.size(m_fname.c_str());
if( fsize != m_opts.numBlocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from fs " << fsize << " != " << m_opts.numBlocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
if( returnval )
{
logMsg( INFO, "Running readTest" );
returnval = readTest( filetype );
}
if( returnval )
{
logMsg( INFO, "Running rdwrTest" );
returnval = rdwrTest( filetype );
}
if( returnval && filetype == IDBDataFile::HDFS )
{
logMsg( INFO, "Running hdfsRdwrExhaustTest" );
returnval = hdfsRdwrExhaustTest();
}
if( m_opts.numDbRoots > 0 )
{
logMsg( INFO, "Checking dbroots" );
for( int i = 0; i < m_opts.numDbRoots; ++i )
{
ostringstream dbroot;
dbroot << "/usr/local/Calpont/data" << i+1;
if( !fs.exists(dbroot.str().c_str()) )
{
ostringstream msg;
msg << "Could not locate dbroot directory " << dbroot.str();
logMsg( ERROR, msg.str() );
returnval = false;
}
}
}
list<string> dircontents;
assert( fs.listDirectory( dir.c_str(), dircontents ) == 0 );
ostringstream ldstr;
ldstr << "Listed directory " << dir << ":";
list<string>::iterator iend = dircontents.end();
for( list<string>::iterator i = dircontents.begin(); i != iend; ++i )
{
ldstr << (*i) << ",";
}
logMsg( INFO, ldstr.str() );
assert( dircontents.size() == 1 );
// now check a bogus path and make sure it returns -1
assert( fs.listDirectory( "/this-is-a-bogus-directory", dircontents ) == -1 );
assert( fs.remove( "/this-is-a-bogus-directory" ) == 0 );
assert( !fs.isDir( "/this-is-a-bogus-directory" ));
assert( !fs.isDir( m_fname.c_str() ));
if( returnval )
{
logMsg( INFO, "All tests passed!" );
}
reset();
return returnval;
}
bool TestRunner::flushTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "flushTest" );
reset();
string scratch = "/tmp/rdwr_scratch" + m_fname; // scratch file name if exists
boost::filesystem::remove(scratch);
IDBPolicy::remove(m_fname.c_str());
// scenario: writer opens the file, writes 8 bytes, flush;
// reader opens the file, reads 8 bytes, verifys the data, then closes file;
// writer writes 8M bytes (force to buffered file) if -m option used correctly;
// reader opens the file, verifys the file size and content, then closes the file;
// writer closes the file.
ostringstream errstr;
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts, /*default:4*/ 1);
if (!m_file)
{
errstr << "flushTest: open " << m_fname.c_str() << " for write failed";
logMsg( ERROR, errstr.str() );
return false;
}
// write 8 "1" through mem buff
const char w1[] = "11111111";
ssize_t bytes = 0;
if ((bytes = m_file->write(w1, 8)) != 8)
{
errstr << "flushTest: write count = 8, return = " << bytes;
logMsg( ERROR, errstr.str() );
return false;
}
// for HDFS, force writing out to disk.
m_file->flush();
if (!IDBPolicy::exists(m_fname.c_str()))
{
errstr << "flushTest: " << m_fname.c_str() << " does not exist";
logMsg( ERROR, errstr.str() );
return false;
}
if (filetype == IDBDataFile::HDFS && boost::filesystem::exists(scratch))
{
errstr << "flushTest: " << scratch << " exists after 1st write";
logMsg( ERROR, errstr.str() );
return false;
}
// read from file in "r" mode
IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1);
if (!file)
{
errstr << "flushTest: 1st open " << m_fname.c_str() << " to read failed";
logMsg( ERROR, errstr.str() );
return false;
}
char r1[9] = {0};
ssize_t n1 = file->pread(r1, 0, 8);
if (n1 != 8 || strncmp(r1, w1, 8) != 0)
{
errstr << "flushTest: read " << n1 << " != 8 OR " << r1 << "!= 11111111";
logMsg( ERROR, errstr.str() );
return false;
}
delete file;
file = NULL;
// write 8M "2", switched to file buffer if max size for mem buffer is small.
//char w2[] = {[0 ... 8*1024*1024] = '2'};
ssize_t m9 = 9*1024*1024; // must be large than EXTENTSIZE = 8390656 to swith to file buffer
boost::scoped_array<char> w2(new char[m9]);
memset(w2.get(), '2', m9);
m_file->write(w2.get(), m9);
m_file->flush();
// check file size
if (IDBPolicy::size(m_fname.c_str()) != 8 + m9)
{
errstr << "flushTest: size of " << m_fname.c_str() << " is "
<< IDBPolicy::size(m_fname.c_str()) << ", expecting " << (8 + m9);
logMsg( ERROR, errstr.str() );
return false;
}
if (filetype == IDBDataFile::HDFS &&
!boost::filesystem::exists(scratch) &&
m_opts.hdfsMaxMem < m9)
{
errstr << "flushTest: " << scratch << " does not exist after 2nd write";
logMsg( ERROR, errstr.str() );
return false;
}
// 2nd read
file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1);
if (!file)
{
errstr << "flushTest: 2nd open " << m_fname.c_str() << " to read failed";
logMsg( ERROR, errstr.str() );
return false;
}
//char r2[9*1024*1024 + 8 + 1] = {0};
boost::scoped_array<char> r2(new char[9*1024*1024 + 8 + 1]);
memset(r2.get(), 0, m9 + 9);
ssize_t n2 = file->pread(r2.get(), 0, m9 + 8);
if (n2 != (m9+8) || strncmp(r2.get(), w1, 8) != 0 || memcmp(r2.get()+8, w2.get(), m9) != 0)
{
errstr << "flushTest: 2nd read " << m_fname.c_str() << " failed" << endl
<< " return value: " << n2 << " bytes -- " << r2; // need hex dump?
logMsg( ERROR, errstr.str() );
return false;
}
delete file;
file = NULL;
delete m_file;
m_file = NULL;
return true;
}
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
m_open_opts = open_opts;
ostringstream infostr;
string typeString = ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
infostr << "Running test for file type " << typeString;
logMsg( INFO, infostr.str() );
if (filetype != IDBDataFile::UNBUFFERED && filetype != IDBDataFile::BUFFERED)
{
if( !IDBPolicy::installPlugin(m_opts.pluginFile) )
{
cout << "ERROR: unable to install HDFS plugin!" << endl;
return -1;
}
}
IDBFileSystem& fs = IDBFileSystem::getFs( filetype );
// build the file name we are going to use
ostringstream oss;
// embed pid so that this is a new directory path
oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id << "/Calpont/data";
// we need to make sure this directory doesn't already exist
// todo-this only works non-HDFS
string cmd = "rm -rf " + oss.str();
system(cmd.c_str());
string dir = oss.str();
m_fname = dir + "/foobar";
// instantiate this here so that we always clean up files we created no matter
// where we exit the function from
boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );
// HDFS will automatically create parent directories when opening a file so these
// tests around mkdir are irrelevant.
if( filetype != IDBDataFile::HDFS )
{
// we expect this to fail because the parent directory should not exist
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts);
if( m_file )
{
ostringstream errstr;
errstr << "open for writing of path " << m_fname << " should not succeed!";
logMsg( ERROR, errstr.str() );
return false;
}
// now create the path
if (fs.mkdir(dir.c_str()))
{
ostringstream errstr;
errstr << "open mkdir of " << dir << " failed!";
logMsg( ERROR, errstr.str() );
return false;
}
}
bool returnval = true;
if( returnval )
{
returnval = writeTest( filetype );
}
// going to check the size two different ways - once through the file and
// once through the file system.
unsigned fsize = m_file->size();
if( fsize != m_opts.numblocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from file " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
// this deletes and closes the file - required to get accurate file size with
// buffered IO and in HDFS from the file system
reset();
fsize = fs.size(m_fname.c_str());
if( fsize != m_opts.numblocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from fs " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
if( returnval )
{
returnval = tellTest( filetype );
}
if( returnval )
{
returnval = readTest( filetype );
}
if( returnval && (filetype == IDBDataFile::UNBUFFERED || filetype == IDBDataFile::HDFS ))
{
returnval = concurrencyTest( filetype );
}
if( returnval )
{
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
if( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for reading";
logMsg( ERROR, errstr.str() );
return false;
}
if( m_opts.numthreads > 1)
{
boost::thread_group thread_group;
for( int i = 0; i < m_opts.numthreads; ++i )
{
thread_group.create_thread(boost::bind(thread_func2, boost::ref(*this)));
}
thread_group.join_all();
}
returnval = randomReadTest();
}
if( returnval )
{
returnval = rdwrTest( filetype );
}
if( returnval && filetype == IDBDataFile::BUFFERED )
{
returnval = openByModeStrTest();
}
if( returnval )
{
returnval = truncateTest( filetype );
}
if( returnval )
{
returnval = renameTest( filetype );
}
if( returnval )
{
returnval = copyTest( filetype );
}
if( returnval && filetype == IDBDataFile::HDFS )
{
returnval = hdfsRdwrExhaustTest();
}
if( returnval )
{
returnval = flushTest( filetype );
}
if( returnval )
{
returnval = seekTest( filetype );
}
if( returnval )
{
returnval = listDirTest( filetype, dir );
}
if( returnval )
logMsg( INFO, typeString + " tests passed!", true );
reset();
return returnval;
}
bool TestRunner::seekTest( IDBDataFile::Types filetype )
{
logMsg( INFO, "seekTest" );
reset();
// scenario: writer opens the file with w+, writes 8 bytes, seek to 4 from 0, write 4 bytes
// reader opens the file with r, verify size and contents,
// writer seeks 4 bytes beyond EOF, write 4 bytes, and close the file,
// reader rewinds, verify size and contents, and close the file.
ostringstream errstr;
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts);
if (!m_file)
{
errstr << "seekTest: open " << m_fname.c_str() << " for write failed";
logMsg( ERROR, errstr.str() );
return false;
}
// write 8 "1" through mem buff
const char w1[] = "11111111";
ssize_t bytes = 0;
if ((bytes = m_file->write(w1, 8)) != 8)
{
errstr << "seekTest: write1 count = 8, return = " << bytes;
logMsg( ERROR, errstr.str() );
return false;
}
if (m_file->seek(4, SEEK_SET) != 0)
{
errstr << "seekTest: seek(4, SEEK_SET) failed";
logMsg( ERROR, errstr.str() );
return false;
}
const char w2[] = "2222";
if ((bytes = m_file->write(w2, 4)) != 4)
{
errstr << "seekTest: write2 count = 4, return = " << bytes;
logMsg( ERROR, errstr.str() );
return false;
}
// for HDFS, force writing out to disk.
m_file->flush();
// read from file in "r" mode
IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
if (!file)
{
errstr << "seekTest: 1st open " << m_fname.c_str() << " to read failed";
logMsg( ERROR, errstr.str() );
return false;
}
char r1[9] = {0};
ssize_t n1 = file->pread(r1, 0, 8);
if (IDBPolicy::size(m_fname.c_str()) != 8)
{
errstr << "seekTest: size of " << m_fname.c_str() << " is "
<< IDBPolicy::size(m_fname.c_str()) << ", expecting 8";
logMsg( ERROR, errstr.str() );
return false;
}
if (n1 != 8 || strncmp(r1, w1, 4) != 0 || strncmp(r1+4, w2, 4) != 0)
{
errstr << "seekTest: read " << n1 << " != 8 OR " << r1 << "!= 11112222";
logMsg( ERROR, errstr.str() );
return false;
}
// now seek beyond the eof, and write 4 bytes.
const char w3[] = "3333";
if (m_file->seek(4, SEEK_END) != 0)
{
errstr << "seekTest: seek(4, SEEK_END) failed";
logMsg( ERROR, errstr.str() );
return false;
}
if ((bytes = m_file->write(w3, 4)) != 4)
{
errstr << "seekTest: write3 count = 4, return = " << bytes;
logMsg( ERROR, errstr.str() );
return false;
}
m_file->flush();
delete m_file;
m_file = NULL;
// check file size
if (IDBPolicy::size(m_fname.c_str()) != 16)
{
errstr << "seekTest: size of " << m_fname.c_str() << " is "
<< IDBPolicy::size(m_fname.c_str()) << ", expecting 16";
logMsg( ERROR, errstr.str() );
return false;
}
// 2nd read
file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
if (!file)
{
errstr << "seekTest: 2nd open " << m_fname.c_str() << " to read failed";
logMsg( ERROR, errstr.str() );
return false;
}
char r2[17] = {0};
const char pd[4] = {0}; // padding
ssize_t n2 = file->pread(r2, 0, 16);
if (n2 != 16 ||
strncmp(r2, w1, 4) != 0 || memcmp(r2+4, w2, 4) != 0 ||
strncmp(r2+8, pd, 4) != 0 || memcmp(r2+12, w3, 4) != 0)
{
errstr << "seekTest: 2nd read " << m_fname.c_str() << " failed" << endl
<< " return value: " << n2 << " bytes -- " << r2; // need hex dump?
logMsg( ERROR, errstr.str() );
return false;
}
delete file;
file = NULL;
return true;
}