bool TestRunner::tellTest( IDBDataFile::Types filetype ) { logMsg( INFO, "tellTest" ); reset(); // scenario: reader opens file, seeks somewhere and tells where it is. m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); assert(m_file); // read a few blocks assert( readBlock(0, m_defbuf, 0) ); assert( readBlock(1, m_defbuf, 1) ); if( m_file->seek(BLK_SIZE, SEEK_SET) ) { ostringstream errstr; errstr << "tellTest: failed to seek block"; logMsg( ERROR, errstr.str() ); return false; } off64_t filepos = m_file->tell(); if( filepos != off64_t(BLK_SIZE) ) { ostringstream errstr; errstr << "tellTest: File position not at correct block, " << filepos << " != " << BLK_SIZE; logMsg( ERROR, errstr.str() ); return false; } return true; }
bool TestRunner::concurrencyTest( IDBDataFile::Types filetype ) { logMsg( INFO, "concurrencyTest" ); reset(); // ok - scenario is a reader opens the file then a reader/writer opens // the same file and updates something, then the reader tries to read // again. m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); assert(m_file); // read a few blocks assert( readBlock(0, m_defbuf, 0) ); assert( readBlock(1, m_defbuf, 1) ); // open the same file for read/write IDBDataFile* rdwrFile = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); assert(rdwrFile); // read a block, write a block assert( rdwrFile->pread(m_defbuf, 0, BLK_SIZE) == (ssize_t) BLK_SIZE ); assert( m_defbuf[0] == 0 ); m_defbuf[0] = 95; assert( rdwrFile->seek(0, 0) == 0 ); assert( rdwrFile->write(m_defbuf, BLK_SIZE) ); // close file delete rdwrFile; // this 5 seconds is important in the HDFS case because it gives HDFS // time to age off (or whatever) the blocks for the original file that // have now been rewritten. The value was determined experimentally - // 3 secs works fine most times but not all. If HDFS hasn't aged out // the blocks then the read will return the old data if( filetype == IDBDataFile::HDFS ) sleep(10); // go back to the reader and make sure he got the new value, then close assert( readBlock(0, m_defbuf, 95) ); delete m_file; // now put block 0 back the way it was m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); assert(m_file); assert( writeBlock(0, m_defbuf, 0) ); return true; }
bool TestRunner::hdfsRdwrExhaustTest() { // this is going to be a self-contained test that attempts to test // all logic inherent in HdfsRdwr // choose a new filename that is specific to our thread ostringstream oss; // embed pid so that this is a new directory path oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id; string newpath = oss.str(); // open a file with arbitrarily small buffer IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8); assert( file ); // check various empty file conditions assert( file->size() == 0 ); assert( file->tell() == 0 ); assert( file->seek(-1, SEEK_CUR) == -1); assert( file->seek(0, SEEK_SET) == 0); unsigned char buf[4]; assert( file->read(buf, 4) == 0); // write some data buf[0] = 0xde; buf[1] = 0xad; buf[2] = 0xbe; buf[3] = 0xef; assert( file->write(buf, 4) == 4); assert( file->size() == 4 ); assert( file->tell() == 4 ); assert( file->truncate(-1) == -1 ); // now make file empty again assert( file->truncate(0) == 0 ); assert( file->size() == 0 ); assert( file->seek(0, SEEK_SET) == 0); assert( file->tell() == 0 ); assert( file->read(buf, 4) == 0); // write data again, this time exactly up to allocated size assert( file->write(buf, 4) == 4); assert( file->write(buf, 4) == 4); assert( file->size() == 8 ); assert( file->tell() == 8 ); // truncate back to 4 assert( file->truncate(4) == 0 ); assert( file->size() == 4 ); assert( file->seek(4, SEEK_SET) == 0); assert( file->tell() == 4 ); // now trigger a buffer reallocation assert( file->write(buf, 4) == 4); assert( file->write(buf, 4) == 4); assert( file->size() == 12 ); // now delete and close. delete file; // check the file size through the file system IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS ); assert( fs.size( newpath.c_str() ) == 12); // open again - the file is bigger than the default buffer so it triggers alternate // logic in the constructor file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8); assert( file ); assert( file->size() == 12); unsigned char newbuf[4]; assert( file->pread(newbuf, 4, 4) == 4); assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe &&newbuf[3] == 0xef); delete file; fs.remove(newpath.c_str()); return true; }
bool TestRunner::rdwrTest( IDBDataFile::Types filetype ) { reset(); m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); if( !m_file ) { ostringstream errstr; errstr << "Unable to open " << m_fname << " for reading"; logMsg( ERROR, errstr.str() ); return false; } struct drand48_data d48data; srand48_r(0xdeadbeef, &d48data); // we will write to 5 random blocks and then come back and // verify the contents and undo them int blocks_to_touch = min( 5, m_opts.numBlocks); vector<int> touched; for( int i = 0; i < blocks_to_touch; ++i ) { long int blk_num; // we need to make sure all the blocks we touch are unique or // the pattern used by this test won't work bool found = false; while(!found) { lrand48_r( &d48data, &blk_num); blk_num = blk_num % m_opts.numBlocks; vector<int>::iterator pos = find( touched.begin(), touched.end(), blk_num ); if( pos == touched.end()) found = true; } if( m_file->seek(blk_num * BLK_SIZE, SEEK_SET) ) { ostringstream errstr; errstr << "failed to seek block " << blk_num; logMsg( ERROR, errstr.str() ); return false; } unsigned char writeval = 0xb0; size_t writect = m_file->write(&writeval,1); if( writect != 1 ) { ostringstream errstr; errstr << "Only wrote " << writect << " bytes, expected 1"; logMsg( ERROR, errstr.str() ); return false; } touched.push_back(blk_num); } m_file->flush(); for( int i = 0; i < (int) touched.size(); ++i ) { unsigned char readbuf; size_t readct = m_file->pread(&readbuf,touched[i] * BLK_SIZE,1); if( readct != 1 || readbuf != (unsigned char) 0xb0 ) { ostringstream errstr; errstr << "Error reading expected value, readct=" << readct << " bytes, value" << (int) readbuf; logMsg( ERROR, errstr.str() ); return false; } readbuf = touched[i]; if( m_file->seek(-1, SEEK_CUR) ) { ostringstream errstr; errstr << "failed to seek -1"; logMsg( ERROR, errstr.str() ); return false; } size_t writect = m_file->write(&readbuf,1); if( writect != 1 ) { ostringstream errstr; errstr << "Only wrote " << writect << " bytes, expected 1"; logMsg( ERROR, errstr.str() ); return false; } } return true; }
bool TestRunner::seekTest( IDBDataFile::Types filetype ) { logMsg( INFO, "seekTest" ); reset(); // scenario: writer opens the file with w+, writes 8 bytes, seek to 4 from 0, write 4 bytes // reader opens the file with r, verify size and contents, // writer seeks 4 bytes beyond EOF, write 4 bytes, and close the file, // reader rewinds, verify size and contents, and close the file. ostringstream errstr; m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts); if (!m_file) { errstr << "seekTest: open " << m_fname.c_str() << " for write failed"; logMsg( ERROR, errstr.str() ); return false; } // write 8 "1" through mem buff const char w1[] = "11111111"; ssize_t bytes = 0; if ((bytes = m_file->write(w1, 8)) != 8) { errstr << "seekTest: write1 count = 8, return = " << bytes; logMsg( ERROR, errstr.str() ); return false; } if (m_file->seek(4, SEEK_SET) != 0) { errstr << "seekTest: seek(4, SEEK_SET) failed"; logMsg( ERROR, errstr.str() ); return false; } const char w2[] = "2222"; if ((bytes = m_file->write(w2, 4)) != 4) { errstr << "seekTest: write2 count = 4, return = " << bytes; logMsg( ERROR, errstr.str() ); return false; } // for HDFS, force writing out to disk. m_file->flush(); // read from file in "r" mode IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); if (!file) { errstr << "seekTest: 1st open " << m_fname.c_str() << " to read failed"; logMsg( ERROR, errstr.str() ); return false; } char r1[9] = {0}; ssize_t n1 = file->pread(r1, 0, 8); if (IDBPolicy::size(m_fname.c_str()) != 8) { errstr << "seekTest: size of " << m_fname.c_str() << " is " << IDBPolicy::size(m_fname.c_str()) << ", expecting 8"; logMsg( ERROR, errstr.str() ); return false; } if (n1 != 8 || strncmp(r1, w1, 4) != 0 || strncmp(r1+4, w2, 4) != 0) { errstr << "seekTest: read " << n1 << " != 8 OR " << r1 << "!= 11112222"; logMsg( ERROR, errstr.str() ); return false; } // now seek beyond the eof, and write 4 bytes. const char w3[] = "3333"; if (m_file->seek(4, SEEK_END) != 0) { errstr << "seekTest: seek(4, SEEK_END) failed"; logMsg( ERROR, errstr.str() ); return false; } if ((bytes = m_file->write(w3, 4)) != 4) { errstr << "seekTest: write3 count = 4, return = " << bytes; logMsg( ERROR, errstr.str() ); return false; } m_file->flush(); delete m_file; m_file = NULL; // check file size if (IDBPolicy::size(m_fname.c_str()) != 16) { errstr << "seekTest: size of " << m_fname.c_str() << " is " << IDBPolicy::size(m_fname.c_str()) << ", expecting 16"; logMsg( ERROR, errstr.str() ); return false; } // 2nd read file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); if (!file) { errstr << "seekTest: 2nd open " << m_fname.c_str() << " to read failed"; logMsg( ERROR, errstr.str() ); return false; } char r2[17] = {0}; const char pd[4] = {0}; // padding ssize_t n2 = file->pread(r2, 0, 16); if (n2 != 16 || strncmp(r2, w1, 4) != 0 || memcmp(r2+4, w2, 4) != 0 || strncmp(r2+8, pd, 4) != 0 || memcmp(r2+12, w3, 4) != 0) { errstr << "seekTest: 2nd read " << m_fname.c_str() << " failed" << endl << " return value: " << n2 << " bytes -- " << r2; // need hex dump? logMsg( ERROR, errstr.str() ); return false; } delete file; file = NULL; return true; }