Example #1
0
bool TestRunner::concurrencyTest( IDBDataFile::Types filetype )
{
    logMsg( INFO, "concurrencyTest" );

    reset();

    // ok - scenario is a reader opens the file then a reader/writer opens
    // the same file and updates something, then the reader tries to read
    // again.
    m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
    assert(m_file);

    // read a few blocks
    assert( readBlock(0, m_defbuf, 0) );
    assert( readBlock(1, m_defbuf, 1) );

    // open the same file for read/write
    IDBDataFile* rdwrFile = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
    assert(rdwrFile);

    // read a block, write a block
    assert( rdwrFile->pread(m_defbuf, 0, BLK_SIZE) == (ssize_t) BLK_SIZE );
    assert( m_defbuf[0] == 0 );
    m_defbuf[0] = 95;
    assert( rdwrFile->seek(0, 0) == 0 );
    assert( rdwrFile->write(m_defbuf, BLK_SIZE) );

    // close file
    delete rdwrFile;

    // this 5 seconds is important in the HDFS case because it gives HDFS
    // time to age off (or whatever) the blocks for the original file that
    // have now been rewritten.  The value was determined experimentally -
    // 3 secs works fine most times but not all.  If HDFS hasn't aged out
    // the blocks then the read will return the old data
    if( filetype == IDBDataFile::HDFS )
        sleep(10);

    // go back to the reader and make sure he got the new value, then close
    assert( readBlock(0, m_defbuf, 95) );
    delete m_file;

    // now put block 0 back the way it was
    m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
    assert(m_file);

    assert( writeBlock(0, m_defbuf, 0) );

    return true;
}
Example #2
0
bool TestRunner::randomReadTest()
{
    logMsg( INFO, "randomReadTest" );

    struct drand48_data d48data;
    srand48_r(pthread_self(), &d48data);

    for( int i = 0; i < 10; ++i )
    {
        long int blk_num;
        lrand48_r( &d48data, &blk_num);
        blk_num = blk_num % m_opts.numblocks;

        unsigned char readbuf[4];
        assert( m_file->pread( readbuf, blk_num * BLK_SIZE, 4 ) == 4 );
        assert( readbuf[0] == (unsigned char) blk_num );
    }

    return true;
}
Example #3
0
bool TestRunner::readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag)
{
	size_t rc = m_file->pread(buf,blocknum*BLK_SIZE,BLK_SIZE);
	//cout << "DEBUG: read " << rc << " bytes at offset " << blocknum * BLK_SIZE << endl;

	if (rc != BLK_SIZE)
	{
		ostringstream errstr;
		errstr << "readBlock failed for block " << blocknum << ", read " << rc << " bytes, expecting " << BLK_SIZE;
		logMsg( ERROR, errstr.str() );
		return false;
	}
	else if (tag != buf[0])
	{
		ostringstream errstr;
		errstr << "read tag 0x" << setw(2) << hex << setfill('0') << (int) buf[0] << " at block " << dec
				<< blocknum << ", expected 0x" << (int) tag;
		logMsg( ERROR, errstr.str() );
		return false;
	}

	return true;
}
Example #4
0
bool TestRunner::hdfsRdwrExhaustTest()
{
	// this is going to be a self-contained test that attempts to test
	// all logic inherent in HdfsRdwr

	// choose a new filename that is specific to our thread
	ostringstream oss;
	// embed pid so that this is a new directory path
	oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id;
	string newpath = oss.str();

	// open a file with arbitrarily small buffer
	IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
	assert( file );

	// check various empty file conditions
	assert( file->size() == 0 );
	assert( file->tell() == 0 );
	assert( file->seek(-1, SEEK_CUR) == -1);
	assert( file->seek(0, SEEK_SET) == 0);
	unsigned char buf[4];
	assert( file->read(buf, 4) == 0);

	// write some data
	buf[0] = 0xde; buf[1] = 0xad; buf[2] = 0xbe; buf[3] = 0xef;
	assert( file->write(buf, 4) == 4);
	assert( file->size() == 4 );
	assert( file->tell() == 4 );
	assert( file->truncate(-1) == -1 );

	// now make file empty again
	assert( file->truncate(0) == 0 );
	assert( file->size() == 0 );
	assert( file->seek(0, SEEK_SET) == 0);
	assert( file->tell() == 0 );
	assert( file->read(buf, 4) == 0);

	// write data again, this time exactly up to allocated size
	assert( file->write(buf, 4) == 4);
	assert( file->write(buf, 4) == 4);
	assert( file->size() == 8 );
	assert( file->tell() == 8 );

	// truncate back to 4
	assert( file->truncate(4) == 0 );
	assert( file->size() == 4 );
	assert( file->seek(4, SEEK_SET) == 0);
	assert( file->tell() == 4 );

	// now trigger a buffer reallocation
	assert( file->write(buf, 4) == 4);
	assert( file->write(buf, 4) == 4);
	assert( file->size() == 12 );

	// now delete and close.
	delete file;

	// check the file size through the file system
	IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS );
	assert( fs.size( newpath.c_str() ) == 12);

	// open again - the file is bigger than the default buffer so it triggers alternate
	// logic in the constructor
	file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
	assert( file );
	assert( file->size() == 12);
	unsigned char newbuf[4];
	assert( file->pread(newbuf, 4, 4) == 4);
	assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe &&newbuf[3] == 0xef);
	delete file;

	fs.remove(newpath.c_str());

	return true;
}
Example #5
0
bool TestRunner::rdwrTest( IDBDataFile::Types filetype )
{
	reset();
	m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
	if( !m_file )
	{
		ostringstream errstr;
		errstr << "Unable to open " << m_fname << " for reading";
		logMsg( ERROR, errstr.str() );
		return false;
	}

	struct drand48_data d48data;
    srand48_r(0xdeadbeef, &d48data);

    // we will write to 5 random blocks and then come back and
    // verify the contents and undo them
    int blocks_to_touch = min( 5, m_opts.numBlocks);
    vector<int> touched;
    for( int i = 0; i < blocks_to_touch; ++i )
    {
    	long int blk_num;
    	// we need to make sure all the blocks we touch are unique or
    	// the pattern used by this test won't work
    	bool found = false;
    	while(!found)
    	{
    		lrand48_r( &d48data, &blk_num);
    		blk_num = blk_num % m_opts.numBlocks;
    		vector<int>::iterator pos = find( touched.begin(), touched.end(), blk_num );
    		if( pos == touched.end())
    			found = true;
    	}

    	if( m_file->seek(blk_num * BLK_SIZE, SEEK_SET) )
    	{
    		ostringstream errstr;
    		errstr << "failed to seek block " << blk_num;
    		logMsg( ERROR, errstr.str() );
    		return false;
    	}

    	unsigned char writeval = 0xb0;
    	size_t writect = m_file->write(&writeval,1);
       	if( writect != 1 )
    	{
    		ostringstream errstr;
    		errstr << "Only wrote " << writect << " bytes, expected 1";
    		logMsg( ERROR, errstr.str() );
    		return false;
    	}

       	touched.push_back(blk_num);
    }

    m_file->flush();

    for( int i = 0; i < (int) touched.size(); ++i )
    {
    	unsigned char readbuf;
    	size_t readct = m_file->pread(&readbuf,touched[i] * BLK_SIZE,1);
    	if( readct != 1 || readbuf != (unsigned char) 0xb0 )
    	{
    		ostringstream errstr;
    		errstr << "Error reading expected value, readct=" << readct << " bytes, value" << (int) readbuf;
    		logMsg( ERROR, errstr.str() );
    		return false;
    	}

    	readbuf = touched[i];

    	if( m_file->seek(-1, SEEK_CUR) )
    	{
    		ostringstream errstr;
    		errstr << "failed to seek -1";
    		logMsg( ERROR, errstr.str() );
    		return false;
    	}

    	size_t writect = m_file->write(&readbuf,1);
       	if( writect != 1 )
    	{
    		ostringstream errstr;
    		errstr << "Only wrote " << writect << " bytes, expected 1";
    		logMsg( ERROR, errstr.str() );
    		return false;
    	}
   }

    return true;
}
Example #6
0
bool TestRunner::flushTest( IDBDataFile::Types filetype )
{
    logMsg( INFO, "flushTest" );

    reset();

    string scratch = "/tmp/rdwr_scratch" + m_fname;  // scratch file name if exists
    boost::filesystem::remove(scratch);
    IDBPolicy::remove(m_fname.c_str());

    // scenario: writer opens the file, writes 8 bytes, flush;
    //           reader opens the file, reads 8 bytes, verifys the data, then closes file;
    //           writer writes 8M bytes (force to buffered file) if -m option used correctly;
    //           reader opens the file, verifys the file size and content, then closes the file;
    //           writer closes the file.
    ostringstream errstr;
    m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts, /*default:4*/ 1);
    if (!m_file)
    {
        errstr << "flushTest: open " << m_fname.c_str() << " for write failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // write 8 "1" through mem buff
    const char w1[] = "11111111";
    ssize_t bytes = 0;
    if ((bytes = m_file->write(w1, 8)) != 8)
    {
        errstr << "flushTest: write count = 8, return = " << bytes;
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // for HDFS, force writing out to disk.
    m_file->flush();
    if (!IDBPolicy::exists(m_fname.c_str()))
    {
        errstr << "flushTest: " << m_fname.c_str() << " does not exist";
        logMsg( ERROR, errstr.str() );
        return false;
    }
    if (filetype == IDBDataFile::HDFS && boost::filesystem::exists(scratch))
    {
        errstr << "flushTest: " << scratch << " exists after 1st write";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // read from file in "r" mode
    IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1);
    if (!file)
    {
        errstr << "flushTest: 1st open " << m_fname.c_str() << " to read failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    char r1[9] = {0};
    ssize_t n1 = file->pread(r1, 0, 8);
    if (n1 != 8 || strncmp(r1, w1, 8) != 0)
    {
        errstr << "flushTest: read " << n1 << " != 8 OR " << r1 << "!= 11111111";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    delete file;
    file = NULL;

    // write 8M "2", switched to file buffer if max size for mem buffer is small.
    //char w2[] = {[0 ... 8*1024*1024] = '2'};
    ssize_t m9 = 9*1024*1024;  // must be large than EXTENTSIZE = 8390656 to swith to file buffer
    boost::scoped_array<char> w2(new char[m9]);
    memset(w2.get(), '2', m9);
    m_file->write(w2.get(), m9);
    m_file->flush();

    // check file size
    if (IDBPolicy::size(m_fname.c_str()) != 8 + m9)
    {
        errstr << "flushTest: size of " << m_fname.c_str() << " is "
               << IDBPolicy::size(m_fname.c_str()) << ", expecting " << (8 + m9);
        logMsg( ERROR, errstr.str() );
        return false;
    }
    if (filetype == IDBDataFile::HDFS &&
            !boost::filesystem::exists(scratch) &&
            m_opts.hdfsMaxMem < m9)
    {
        errstr << "flushTest: " << scratch << " does not exist after 2nd write";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // 2nd read
    file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1);
    if (!file)
    {
        errstr << "flushTest: 2nd open " << m_fname.c_str() << " to read failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    //char r2[9*1024*1024 + 8 + 1] = {0};
    boost::scoped_array<char> r2(new char[9*1024*1024 + 8 + 1]);
    memset(r2.get(), 0, m9 + 9);
    ssize_t n2 = file->pread(r2.get(), 0, m9 + 8);
    if (n2 != (m9+8) || strncmp(r2.get(), w1, 8) != 0 || memcmp(r2.get()+8, w2.get(), m9) != 0)
    {
        errstr << "flushTest: 2nd read " << m_fname.c_str() << " failed" << endl
               << "   return value: " << n2 << " bytes -- " << r2;  // need hex dump?
        logMsg( ERROR, errstr.str() );
        return false;
    }

    delete file;
    file = NULL;

    delete m_file;
    m_file = NULL;

    return true;
}
Example #7
0
bool TestRunner::seekTest( IDBDataFile::Types filetype )
{
    logMsg( INFO, "seekTest" );

    reset();

    // scenario: writer opens the file with w+, writes 8 bytes, seek to 4 from 0, write 4 bytes
    //           reader opens the file with r, verify size and contents,
    //           writer seeks 4 bytes beyond EOF, write 4 bytes, and close the file,
    //           reader rewinds, verify size and contents, and close the file.
    ostringstream errstr;
    m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts);
    if (!m_file)
    {
        errstr << "seekTest: open " << m_fname.c_str() << " for write failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // write 8 "1" through mem buff
    const char w1[] = "11111111";
    ssize_t bytes = 0;
    if ((bytes = m_file->write(w1, 8)) != 8)
    {
        errstr << "seekTest: write1 count = 8, return = " << bytes;
        logMsg( ERROR, errstr.str() );
        return false;
    }
    if (m_file->seek(4, SEEK_SET) != 0)
    {
        errstr << "seekTest: seek(4, SEEK_SET) failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }
    const char w2[] = "2222";
    if ((bytes = m_file->write(w2, 4)) != 4)
    {
        errstr << "seekTest: write2 count = 4, return = " << bytes;
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // for HDFS, force writing out to disk.
    m_file->flush();

    // read from file in "r" mode
    IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
    if (!file)
    {
        errstr << "seekTest: 1st open " << m_fname.c_str() << " to read failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    char r1[9] = {0};
    ssize_t n1 = file->pread(r1, 0, 8);
    if (IDBPolicy::size(m_fname.c_str()) != 8)
    {
        errstr << "seekTest: size of " << m_fname.c_str() << " is "
               << IDBPolicy::size(m_fname.c_str()) << ", expecting 8";
        logMsg( ERROR, errstr.str() );
        return false;
    }
    if (n1 != 8 || strncmp(r1, w1, 4) != 0 || strncmp(r1+4, w2, 4) != 0)
    {
        errstr << "seekTest: read " << n1 << " != 8 OR " << r1 << "!= 11112222";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // now seek beyond the eof, and write 4 bytes.
    const char w3[] = "3333";
    if (m_file->seek(4, SEEK_END) != 0)
    {
        errstr << "seekTest: seek(4, SEEK_END) failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }
    if ((bytes = m_file->write(w3, 4)) != 4)
    {
        errstr << "seekTest: write3 count = 4, return = " << bytes;
        logMsg( ERROR, errstr.str() );
        return false;
    }
    m_file->flush();

    delete m_file;
    m_file = NULL;

    // check file size
    if (IDBPolicy::size(m_fname.c_str()) != 16)
    {
        errstr << "seekTest: size of " << m_fname.c_str() << " is "
               << IDBPolicy::size(m_fname.c_str()) << ", expecting 16";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // 2nd read
    file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
    if (!file)
    {
        errstr << "seekTest: 2nd open " << m_fname.c_str() << " to read failed";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    char r2[17] = {0};
    const char pd[4] = {0};  // padding
    ssize_t n2 = file->pread(r2, 0, 16);
    if (n2 != 16 ||
            strncmp(r2, w1, 4) != 0 || memcmp(r2+4, w2, 4) != 0 ||
            strncmp(r2+8, pd, 4) != 0 || memcmp(r2+12, w3, 4) != 0)
    {
        errstr << "seekTest: 2nd read " << m_fname.c_str() << " failed" << endl
               << "   return value: " << n2 << " bytes -- " << r2;  // need hex dump?
        logMsg( ERROR, errstr.str() );
        return false;
    }

    delete file;
    file = NULL;

    return true;
}