Exemplo n.º 1
0
bool TestRunner::truncateTest( IDBDataFile::Types filetype )
{
    logMsg( INFO, "truncateTest" );

    reset();
    m_file = IDBDataFile::open(filetype, m_fname.c_str(), "a", m_open_opts);
    if( !m_file )
    {
        ostringstream errstr;
        errstr << "Unable to open " << m_fname << " for writing";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // choose a random block to truncate at
    struct drand48_data d48data;
    srand48_r(0xdeadbeef, &d48data);

    long int blk_num;
    lrand48_r( &d48data, &blk_num);
    blk_num = blk_num % m_opts.numblocks;
    // always leave at least one block
    if( blk_num == 0 )
        blk_num = 1;

    int rc = m_file->truncate(blk_num * BLK_SIZE);
    if( (filetype != IDBDataFile::HDFS) && rc)
    {
        logMsg( ERROR, "truncate failed!" );
        return false;
    }
    else if ( (filetype == IDBDataFile::HDFS) && !rc )
    {
        logMsg( ERROR, "truncate is supposed to fail for HDFS files!" );
        return false;
    }
    else if( (filetype == IDBDataFile::HDFS) )
    {
        // this is the "success" case for HDFS we didn't expect to truncate so reset blk_num
        blk_num = m_opts.numblocks;
    }

    off64_t fsize = m_file->size();
    if( fsize != (off64_t) (blk_num * BLK_SIZE))
    {
        ostringstream errstr;
        errstr << "wrong file size after truncate, " << fsize << " != " << blk_num*BLK_SIZE;
        logMsg( ERROR, errstr.str() );
        return false;
    }

    return true;
}
Exemplo n.º 2
0
bool TestRunner::hdfsRdwrExhaustTest()
{
	// this is going to be a self-contained test that attempts to test
	// all logic inherent in HdfsRdwr

	// choose a new filename that is specific to our thread
	ostringstream oss;
	// embed pid so that this is a new directory path
	oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id;
	string newpath = oss.str();

	// open a file with arbitrarily small buffer
	IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
	assert( file );

	// check various empty file conditions
	assert( file->size() == 0 );
	assert( file->tell() == 0 );
	assert( file->seek(-1, SEEK_CUR) == -1);
	assert( file->seek(0, SEEK_SET) == 0);
	unsigned char buf[4];
	assert( file->read(buf, 4) == 0);

	// write some data
	buf[0] = 0xde; buf[1] = 0xad; buf[2] = 0xbe; buf[3] = 0xef;
	assert( file->write(buf, 4) == 4);
	assert( file->size() == 4 );
	assert( file->tell() == 4 );
	assert( file->truncate(-1) == -1 );

	// now make file empty again
	assert( file->truncate(0) == 0 );
	assert( file->size() == 0 );
	assert( file->seek(0, SEEK_SET) == 0);
	assert( file->tell() == 0 );
	assert( file->read(buf, 4) == 0);

	// write data again, this time exactly up to allocated size
	assert( file->write(buf, 4) == 4);
	assert( file->write(buf, 4) == 4);
	assert( file->size() == 8 );
	assert( file->tell() == 8 );

	// truncate back to 4
	assert( file->truncate(4) == 0 );
	assert( file->size() == 4 );
	assert( file->seek(4, SEEK_SET) == 0);
	assert( file->tell() == 4 );

	// now trigger a buffer reallocation
	assert( file->write(buf, 4) == 4);
	assert( file->write(buf, 4) == 4);
	assert( file->size() == 12 );

	// now delete and close.
	delete file;

	// check the file size through the file system
	IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS );
	assert( fs.size( newpath.c_str() ) == 12);

	// open again - the file is bigger than the default buffer so it triggers alternate
	// logic in the constructor
	file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
	assert( file );
	assert( file->size() == 12);
	unsigned char newbuf[4];
	assert( file->pread(newbuf, 4, 4) == 4);
	assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe &&newbuf[3] == 0xef);
	delete file;

	fs.remove(newpath.c_str());

	return true;
}
Exemplo n.º 3
0
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
	m_open_opts = open_opts;
	ostringstream infostr;
	infostr << "Running test for file type "
			<< ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
					( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
	logMsg( INFO, infostr.str() );

	IDBFileSystem& fs = IDBFileSystem::getFs( filetype );

	// build the file name we are going to use
	ostringstream oss;
	// embed pid so that this is a new directory path
	oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id;
	string dir = oss.str();
	m_fname = dir + "/foobar";

	// instantiate this here so that we always clean up files we created no matter
	// where we exit the function from
	boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );

	logMsg( INFO, "Running writeTest" );
	bool returnval = true;
	returnval = writeTest( filetype );

	if( !returnval )
	{
		return false;
	}

	// going to check the size two different ways - once through the file and
	// once through the file system.
	unsigned fsize = m_file->size();
	if( fsize != m_opts.numBlocks * BLK_SIZE )
	{
		ostringstream errstr;
		errstr << "bad file size from file " << fsize << " != " << m_opts.numBlocks * BLK_SIZE << "!";
		logMsg( ERROR, errstr.str() );
		return false;
	}

	// this deletes and closes the file - required to get accurate file size with
	// buffered IO and in HDFS from the file system
	reset();
	fsize = fs.size(m_fname.c_str());
	if( fsize != m_opts.numBlocks * BLK_SIZE )
	{
		ostringstream errstr;
		errstr << "bad file size from fs " << fsize << " != " << m_opts.numBlocks * BLK_SIZE << "!";
		logMsg( ERROR, errstr.str() );
		return false;
	}

	if( returnval )
	{
		logMsg( INFO, "Running readTest" );
		returnval = readTest( filetype );
	}

	if( returnval )
	{
		logMsg( INFO, "Running rdwrTest" );
		returnval = rdwrTest( filetype );
	}

	if( returnval && filetype == IDBDataFile::HDFS )
	{
		logMsg( INFO, "Running hdfsRdwrExhaustTest" );
		returnval = hdfsRdwrExhaustTest();
	}

	if( m_opts.numDbRoots > 0 )
	{
		logMsg( INFO, "Checking dbroots" );
		for( int i = 0; i < m_opts.numDbRoots; ++i )
		{
			ostringstream dbroot;
			dbroot << "/usr/local/Calpont/data" << i+1;
			if( !fs.exists(dbroot.str().c_str()) )
			{
				ostringstream msg;
				msg << "Could not locate dbroot directory " << dbroot.str();
				logMsg( ERROR, msg.str() );
				returnval = false;
			}
		}
	}

	list<string> dircontents;
	assert( fs.listDirectory( dir.c_str(), dircontents ) == 0 );
	ostringstream ldstr;
	ldstr << "Listed directory " << dir << ":";
	list<string>::iterator iend = dircontents.end();
	for( list<string>::iterator i = dircontents.begin(); i != iend; ++i )
	{
		ldstr << (*i) << ",";
	}
	logMsg( INFO, ldstr.str() );
	assert( dircontents.size() == 1 );

	// now check a bogus path and make sure it returns -1
	assert( fs.listDirectory( "/this-is-a-bogus-directory", dircontents ) == -1 );
	assert( fs.remove( "/this-is-a-bogus-directory" ) == 0 );
	assert( !fs.isDir( "/this-is-a-bogus-directory" ));
	assert( !fs.isDir( m_fname.c_str() ));

	if( returnval )
	{
		logMsg( INFO, "All tests passed!" );
	}

	reset();
	return returnval;
}
Exemplo n.º 4
0
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
    m_open_opts = open_opts;
    ostringstream infostr;
    string typeString = ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
                          ( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
    infostr << "Running test for file type " << typeString;
    logMsg( INFO, infostr.str() );

    if (filetype != IDBDataFile::UNBUFFERED && filetype != IDBDataFile::BUFFERED)
    {
        if( !IDBPolicy::installPlugin(m_opts.pluginFile) )
        {
            cout << "ERROR: unable to install HDFS plugin!" << endl;
            return -1;
        }
    }

    IDBFileSystem& fs = IDBFileSystem::getFs( filetype );

    // build the file name we are going to use
    ostringstream oss;
    // embed pid so that this is a new directory path
    oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id << "/Calpont/data";
    // we need to make sure this directory doesn't already exist
    // todo-this only works non-HDFS
    string cmd = "rm -rf " + oss.str();
    system(cmd.c_str());
    string dir = oss.str();
    m_fname = dir + "/foobar";

    // instantiate this here so that we always clean up files we created no matter
    // where we exit the function from
    boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );

    // HDFS will automatically create parent directories when opening a file so these
    // tests around mkdir are irrelevant.
    if( filetype != IDBDataFile::HDFS )
    {
        // we expect this to fail because the parent directory should not exist
        reset();
        m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts);
        if( m_file )
        {
            ostringstream errstr;
            errstr << "open for writing of path " << m_fname << " should not succeed!";
            logMsg( ERROR, errstr.str() );
            return false;
        }

        // now create the path
        if (fs.mkdir(dir.c_str()))
        {
            ostringstream errstr;
            errstr << "open mkdir of " << dir << " failed!";
            logMsg( ERROR, errstr.str() );
            return false;
        }
    }

    bool returnval = true;

    if( returnval )
    {
        returnval = writeTest( filetype );
    }

    // going to check the size two different ways - once through the file and
    // once through the file system.
    unsigned fsize = m_file->size();
    if( fsize != m_opts.numblocks * BLK_SIZE )
    {
        ostringstream errstr;
        errstr << "bad file size from file " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    // this deletes and closes the file - required to get accurate file size with
    // buffered IO and in HDFS from the file system
    reset();
    fsize = fs.size(m_fname.c_str());
    if( fsize != m_opts.numblocks * BLK_SIZE )
    {
        ostringstream errstr;
        errstr << "bad file size from fs " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!";
        logMsg( ERROR, errstr.str() );
        return false;
    }

    if( returnval )
    {
        returnval = tellTest( filetype );
    }

    if( returnval )
    {
        returnval = readTest( filetype );
    }

    if( returnval && (filetype == IDBDataFile::UNBUFFERED || filetype == IDBDataFile::HDFS ))
    {
        returnval = concurrencyTest( filetype );
    }

    if( returnval )
    {
        reset();
        m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
        if( !m_file )
        {
            ostringstream errstr;
            errstr << "Unable to open " << m_fname << " for reading";
            logMsg( ERROR, errstr.str() );
            return false;
        }

        if( m_opts.numthreads > 1)
        {
            boost::thread_group thread_group;
            for( int i = 0; i < m_opts.numthreads; ++i )
            {
                thread_group.create_thread(boost::bind(thread_func2, boost::ref(*this)));
            }
            thread_group.join_all();
        }

        returnval = randomReadTest();
    }

    if( returnval )
    {
        returnval = rdwrTest( filetype );
    }

    if( returnval && filetype == IDBDataFile::BUFFERED )
    {
        returnval = openByModeStrTest();
    }

    if( returnval )
    {
        returnval = truncateTest( filetype );
    }

    if( returnval )
    {
        returnval = renameTest( filetype );
    }

    if( returnval )
    {
        returnval = copyTest( filetype );
    }

    if( returnval && filetype == IDBDataFile::HDFS )
    {
        returnval = hdfsRdwrExhaustTest();
    }

    if( returnval )
    {
        returnval = flushTest( filetype );
    }

    if( returnval )
    {
        returnval = seekTest( filetype );
    }

    if( returnval )
    {
        returnval = listDirTest( filetype, dir );
    }

    if( returnval )
        logMsg( INFO, typeString + " tests passed!", true );

    reset();
    return returnval;
}