void dir_cpi_impl::sync_is_dir (bool & is_dir,
saga::url url)
{
instance_data idata (this);
is_dir = false;
saga::url dir_url(idata->location_);
boost::filesystem::path name (url.get_path(), boost::filesystem::native);
boost::filesystem::path path (idata->location_.get_path(), boost::filesystem::native);
if ( ! name.has_root_path () )
path /= name;
else
path = name;
if(hdfsExists(fs_, path.string().c_str()) == 0)
{
//Check to see if it is a directory
hdfsFileInfo *info;
instance_data idata(this);
info = hdfsGetPathInfo(fs_, path.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_dir = true;
hdfsFreeFileInfo(info, 1);
}
}
// 这个函数返回一个构造好的chunkid和每个chunk的指针
// 这个里面的chunkId肯定是要在blockManager注册然后汇报信息的
// put的话也是会这样的，可以将这个函数中调用put然后统一汇报信息的接口
ChunkInfo BlockManager::loadFromHdfs(string file_name){
// 由此函数得到的<blockId,指针>
ChunkInfo ci;
string file_name_former,file_name_latter;
unsigned pos=file_name.rfind("$");
file_name_former=file_name.substr(0,pos);
file_name_latter=file_name.substr(pos+1,file_name.length());
int offset=atoi(file_name_latter.c_str());
hdfsFS fs=hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port);
hdfsFile readFile=hdfsOpenFile(fs,file_name_former.c_str(),O_RDONLY,0,0,0);
hdfsFileInfo *hdfsfile=hdfsGetPathInfo(fs,file_name_former.c_str());
if(!readFile){
cout<<"open file error"<<endl;
}
unsigned length=0;
length=length+CHUNK_SIZE*offset;
if(length<hdfsfile->mSize){
void *rt=malloc(CHUNK_SIZE); //newmalloc
tSize bytes_num=hdfsPread(fs,readFile,length,rt,CHUNK_SIZE);
ostringstream chunkid;
chunkid<<file_name.c_str()<<"$"<<offset;
// ci.chunkId=chunkid.gestr().c_str();
ci.hook=rt;
}else{
ostringstream chunkid;
chunkid<<file_name.c_str()<<"$"<<offset;
// ci.chunkId=chunkid.str().c_str();
ci.hook=0;
}
hdfsCloseFile(fs,readFile);
hdfsDisconnect(fs);
return ci;
}
bool MaprInputCodedBlockFile::Open(std::string uri) {
CHECK(!is_open_) << "File already open.";
if (!IsValidUri(uri, "maprfs")) {
LOG(ERROR) << "failed to validate uri: " << uri;
return false;
}
std::string scheme, path;
CHECK(ParseUri(uri, &scheme, &path)) << "Invalid uri format: " << uri;
file_ = hdfsOpenFile(fs_, path.c_str(), O_RDONLY, 0, 0, 0);
if (file_ == NULL) {
LOG(ERROR) << "Failed to open file: " << path;
return false;
}
is_open_ = true;
path_ = path;
// Cache file size
hdfsFileInfo* info = hdfsGetPathInfo(fs_, path_.c_str());
CHECK(info);
size_ = info->mSize;
hdfsFreeFileInfo(info, 1);
return true;
}
HDFSChunkReaderIterator::HDFSChunkReaderIterator(const ChunkID& chunk_id, unsigned& chunk_size,const unsigned& block_size)
:ChunkReaderIterator(chunk_id,block_size,chunk_size){
block_buffer_=new Block(block_size_);
fs_=hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port);
hdfs_fd_=hdfsOpenFile(fs_,chunk_id.partition_id.getName().c_str(),O_RDONLY,0,0,0);
if(!hdfs_fd_){
printf("fails to open HDFS file [%s]\n",chunk_id.partition_id.getName().c_str());
number_of_blocks_=0;
}
const unsigned start_pos=start_pos+CHUNK_SIZE*chunk_id_.chunk_off;
if(hdfsSeek(fs_,hdfs_fd_,start_pos)==-1){
printf("fails to set the start offset %d for [%s]\n",start_pos,chunk_id.partition_id.getName().c_str());
number_of_blocks_=0;
}
hdfsFileInfo *file_info=hdfsGetPathInfo(fs_,"/imdb/");// to be refined after communicating with Zhang Lei
if(start_pos+CHUNK_SIZE<file_info->mSize){
number_of_blocks_=CHUNK_SIZE/block_size_;
}
else{
number_of_blocks_=(file_info->mSize-start_pos)/block_size_;
}
hdfsFreeFileInfo(file_info,1);
}
off64_t HdfsFile::size()
{
off64_t ret = 0;
int savedErrno;
if( ( m_flags & O_RDONLY ) != 0 )
{
hdfsFileInfo* fileinfo;
fileinfo = hdfsGetPathInfo(m_fs,m_fname.c_str());
ret = (fileinfo ? fileinfo->mSize : -1);
if( fileinfo )
hdfsFreeFileInfo(fileinfo,1);
}
else
{
// if file is open for either WRITE or APPEND then we know that
// size is always the current file offset since HDFS can only
// write at the end
ret = tell();
}
savedErrno = errno;
if( IDBLogger::isEnabled() )
IDBLogger::logSize(m_fname, this, ret);
errno = savedErrno;
return ret;
}
int BlockManager::loadFromHdfs(const ChunkID& chunk_id, void* const &desc,const unsigned & length){
lock.acquire();
int ret;
int offset=chunk_id.chunk_off;
hdfsFS fs=hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port);
hdfsFile readFile=hdfsOpenFile(fs,chunk_id.partition_id.getPathAndName().c_str(),O_RDONLY,0,0,0);
hdfsFileInfo *hdfsfile=hdfsGetPathInfo(fs,chunk_id.partition_id.getPathAndName().c_str());// to be refined after communicating with Zhang Lei
if(!readFile){
logging_->elog("Fail to open file [%s].Reason:%s",chunk_id.partition_id.getPathAndName().c_str(),strerror(errno));
hdfsDisconnect(fs);
lock.release();
return -1;
}
else{
logging_->log("file [%s] is opened for offset[%d]\n",chunk_id.partition_id.getPathAndName().c_str(),offset);
}
long int start_pos=CHUNK_SIZE*offset;
if(start_pos<hdfsfile->mSize){
ret=hdfsPread(fs,readFile,start_pos,desc,length);
}else{
lock.release();
ret= -1;
}
hdfsCloseFile(fs,readFile);
hdfsDisconnect(fs);
lock.release();
return ret;
}
bool HdfsFileSystem::isDir(const char* pathname) const
{
hdfsFileInfo* fileinfo;
fileinfo = hdfsGetPathInfo(m_fs,pathname);
bool retval = (fileinfo ? fileinfo->mKind == kObjectKindDirectory : false);
if( fileinfo )
hdfsFreeFileInfo(fileinfo,1);
return retval;
}
qioerr hdfs_getlength(void* fl, int64_t* len_out, void* fs)
{
hdfsFileInfo* f_info = NULL;
f_info = hdfsGetPathInfo(to_hdfs_fs(fs)->hfs, to_hdfs_file(fl)->pathnm);
if (f_info == NULL)
QIO_RETURN_CONSTANT_ERROR(EREMOTEIO, "Unable to get length of file in HDFS");
*len_out = f_info->mSize;
return 0;
}
static struct libhdfs_data *libhdfs_data_create(const struct options *opts)
{
struct libhdfs_data *ldata = NULL;
struct hdfsBuilder *builder = NULL;
hdfsFileInfo *pinfo = NULL;
ldata = calloc(1, sizeof(struct libhdfs_data));
if (!ldata) {
fprintf(stderr, "Failed to allocate libhdfs test data.\n");
goto error;
}
builder = hdfsNewBuilder();
if (!builder) {
fprintf(stderr, "Failed to create builder.\n");
goto error;
}
hdfsBuilderSetNameNode(builder, opts->rpc_address);
hdfsBuilderConfSetStr(builder,
"dfs.client.read.shortcircuit.skip.checksum", "true");
ldata->fs = hdfsBuilderConnect(builder);
if (!ldata->fs) {
fprintf(stderr, "Could not connect to default namenode!\n");
goto error;
}
pinfo = hdfsGetPathInfo(ldata->fs, opts->path);
if (!pinfo) {
int err = errno;
fprintf(stderr, "hdfsGetPathInfo(%s) failed: error %d (%s). "
"Attempting to re-create file.\n",
opts->path, err, strerror(err));
if (libhdfs_data_create_file(ldata, opts))
goto error;
} else if (pinfo->mSize != opts->length) {
fprintf(stderr, "hdfsGetPathInfo(%s) failed: length was %lld, "
"but we want length %lld. Attempting to re-create file.\n",
opts->path, (long long)pinfo->mSize, (long long)opts->length);
if (libhdfs_data_create_file(ldata, opts))
goto error;
}
ldata->file = hdfsOpenFile(ldata->fs, opts->path, O_RDONLY, 0, 0, 0);
if (!ldata->file) {
int err = errno;
fprintf(stderr, "hdfsOpenFile(%s) failed: error %d (%s)\n",
opts->path, err, strerror(err));
goto error;
}
ldata->length = opts->length;
return ldata;
error:
if (pinfo)
hdfsFreeFileInfo(pinfo, 1);
if (ldata)
libhdfs_data_free(ldata);
return NULL;
}
qioerr hdfs_get_owners_for_bytes(qio_file_t* file, hdfs_block_byte_map_t** locs, int* out_num_blocks, char** locale_array, int num_locales, off_t start_byte, off_t len)
{
int i;
int j = 0;
int k;
qioerr err = 0;
char* tmp;
int rnd;
int block_count = 0;
hdfs_block_byte_map_t* loc = NULL;
char*** info = NULL;
hdfsFileInfo* f_info = hdfsGetPathInfo(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm);
if (start_byte == 0 && len == -1) // We want the whole thing
info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, f_info->mSize);
else info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, start_byte + len);
while(info[block_count] != NULL) { // Get the number of blocks that we have
block_count++;
}
loc = (hdfs_block_byte_map_t*)qio_calloc(sizeof(hdfs_block_byte_map_t), block_count);
CREATE_ERROR((!info), err, EREMOTEIO, "Unable to get host for HDFS", end);
for (i = 0; info[i] != NULL; i++) { // Assign block owners
rnd = rand() % f_info->mReplication; // pick an owner
if (info[i][rnd]) {// Valid access
tmp = get_locale_name(info[i][rnd]); // strip off .___
for (k = 0; k < num_locales; k++) { // Now find the owner
if (strcmp(tmp, locale_array[k]) == 0) {
loc[i].locale_id = k; // return locale ID for that name
break;
}
}
loc[i].start_byte = (off_t)(i*f_info->mBlockSize);
loc[i].len = (off_t)(f_info->mBlockSize);
j++;
} else {
QIO_GET_CONSTANT_ERROR(err, EINVAL, "Unable to find address for blocks in hdfs_get_owners_for_bytes");
qio_free(loc);
*locs = NULL;
*out_num_blocks = 0;
goto end;
}
}
*locs = loc;
*out_num_blocks = j;
end:
return err;
}
/**
* call-seq:
* hdfs.stat(path) -> file_info
*
* Stats the file or directory at the supplied path, returning a
* Hadoop::DFS::FileInfo object corresponding to it. If this fails, raises a
* DFSException.
*/
VALUE HDFS_File_System_stat(VALUE self, VALUE path) {
FSData* data = get_FSData(self);
hdfsFileInfo* info = hdfsGetPathInfo(data->fs, StringValuePtr(path));
if (info == NULL) {
rb_raise(e_dfs_exception, "Failed to stat file %s: %s",
StringValuePtr(path), get_error(errno));
return Qnil;
}
VALUE file_info = new_HDFS_File_Info(info);
hdfsFreeFileInfo(info, 1);
return file_info;
}
off64_t HdfsFileSystem::size(const char* path) const
{
hdfsFileInfo* fileinfo;
fileinfo = hdfsGetPathInfo(m_fs,path);
off64_t retval = (fileinfo ? fileinfo->mSize : -1);
if( fileinfo )
hdfsFreeFileInfo(fileinfo,1);
if( IDBLogger::isEnabled() )
IDBLogger::logFSop( HDFS, "fs:size", path, this, retval);
return retval;
}
time_t HdfsFile::mtime()
{
boost::mutex::scoped_lock lock(m_mutex);
time_t ret = 0;
hdfsFileInfo* fileinfo;
fileinfo = hdfsGetPathInfo(m_fs,m_fname.c_str());
ret = (fileinfo ? fileinfo->mLastMod : -1);
if( fileinfo )
hdfsFreeFileInfo(fileinfo,1);
return ret;
}
void dir_cpi_impl::sync_remove (saga::impl::void_t & ret,
saga::url url,
int flags)
{
instance_data idata (this);
saga::url dir_url(idata->location_);
boost::filesystem::path src_location (idata->location_.get_path(), boost::filesystem::native);
// complete paths
boost::filesystem::path src_path (url.get_path(), boost::filesystem::native);
if ( ! src_path.has_root_path () )
src_location /= src_path;
else
src_location = src_path;
bool is_src_dir = false;
if(hdfsExists(fs_, src_location.string().c_str()) != 0)
{
SAGA_ADAPTOR_THROW("directory::remove: Can't remove directory: "
"Does not exist", saga::DoesNotExist);
}
else
{
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, src_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_src_dir = true;
else
is_src_dir = false;
hdfsFreeFileInfo(info, 1);
}
if (is_src_dir) {
if (saga::name_space::Recursive != flags)
{
SAGA_ADAPTOR_THROW("directory::remove: Can't remove directory. "
"Please use recursive mode!", saga::BadParameter);
}
else {
saga_hdfs_delete(fs_, src_location.string().c_str());
}
}
else {
saga_hdfs_delete(fs_, src_location.string().c_str());
}
}
int dfs_utimens(const char *path, const struct timespec ts[2])
{
struct hdfsConn *conn = NULL;
hdfsFS fs;
int ret = 0;
dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
TRACE1("utimens", path)
assert(path);
assert(dfs);
assert('/' == *path);
time_t aTime = ts[0].tv_sec;
time_t mTime = ts[1].tv_sec;
ret = fuseConnectAsThreadUid(&conn);
if (ret) {
fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs "
"connection! error %d.\n", ret);
ret = -EIO;
goto cleanup;
}
fs = hdfsConnGetFs(conn);
if (hdfsUtime(fs, path, mTime, aTime)) {
hdfsFileInfo *info = hdfsGetPathInfo(fs, path);
if (info == NULL) {
ret = (errno > 0) ? -errno : -ENOENT;
goto cleanup;
}
// Silently ignore utimens failure for directories, otherwise
// some programs like tar will fail.
if (info->mKind == kObjectKindDirectory) {
ret = 0;
} else {
ret = (errno > 0) ? -errno : -EACCES;
}
goto cleanup;
}
ret = 0;
cleanup:
if (conn) {
hdfsConnRelease(conn);
}
return ret;
}
int dfs_getattr(const char *path, struct stat *st)
{
TRACE1("getattr", path)
// retrieve dfs specific data
dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
// check params and the context var
assert(dfs);
assert(path);
assert(st);
// if not connected, try to connect and fail out if we can't.
if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
syslog(LOG_ERR, "ERROR: could not connect to %s:%d %s:%d\n", dfs->nn_hostname, dfs->nn_port,__FILE__, __LINE__);
return -EIO;
}
// call the dfs API to get the actual information
hdfsFileInfo *info = hdfsGetPathInfo(dfs->fs,path);
if (NULL == info) {
return -ENOENT;
}
fill_stat_structure(&info[0], st);
// setup hard link info - for a file it is 1 else num entries in a dir + 2 (for . and ..)
if (info[0].mKind == kObjectKindDirectory) {
int numEntries = 0;
hdfsFileInfo *info = hdfsListDirectory(dfs->fs,path,&numEntries);
if (info) {
hdfsFreeFileInfo(info,numEntries);
}
st->st_nlink = numEntries + 2;
} else {
// not a directory
st->st_nlink = 1;
}
// free the info pointer
hdfsFreeFileInfo(info,1);
return 0;
}
bool MaprFileSystem::IsDirectory(const std::string& uri){
std::string path = GetUriPathOrDie(uri);
std::string host = "default";
hdfsFS fs = hdfsConnect(host.c_str(), 0); // use default config file settings
CHECK(fs);
hdfsFileInfo* info = hdfsGetPathInfo(fs, path.c_str());
bool is_directory = false;
if (info){
is_directory = (info->mKind == kObjectKindDirectory);
hdfsFreeFileInfo(info,1);
}
else{
LOG(FATAL) << "uri does not exist: " << uri;
}
CHECK_EQ(hdfsDisconnect(fs), 0);
return is_directory;
}
int libhdfsconnector::streamInFile(const char * rfile, int bufferSize)
{
if (!fs)
{
fprintf(stderr, "Could not connect to hdfs on");
return RETURN_FAILURE;
}
unsigned long fileTotalSize = 0;
hdfsFileInfo *fileInfo = NULL;
if ((fileInfo = hdfsGetPathInfo(fs, rfile)) != NULL)
{
fileTotalSize = fileInfo->mSize;
hdfsFreeFileInfo(fileInfo, 1);
}
else
{
fprintf(stderr, "Error: hdfsGetPathInfo for %s - FAILED!\n", rfile);
return RETURN_FAILURE;
}
hdfsFile readFile = hdfsOpenFile(fs, rfile, O_RDONLY, bufferSize, 0, 0);
if (!readFile)
{
fprintf(stderr, "Failed to open %s for writing!\n", rfile);
return RETURN_FAILURE;
}
unsigned char buff[bufferSize + 1];
buff[bufferSize] = '\0';
for (unsigned long bytes_read = 0; bytes_read < fileTotalSize;)
{
unsigned long read_length = hdfsRead(fs, readFile, buff, bufferSize);
bytes_read += read_length;
for (unsigned long i = 0; i < read_length; i++)
fprintf(stdout, "%c", buff[i]);
}
hdfsCloseFile(fs, readFile);
return 0;
}
size_t FileManagerHdfs::numSlots(const block_id block) const {
string filename(blockFilename(block));
hdfsFileInfo *file_info = hdfsGetPathInfo(hdfs_, filename.c_str());
if (file_info == nullptr) {
if (errno != ENOENT) {
LOG_WARNING("Failed to get size of file " << filename << " with error: " << strerror(errno));
}
return 0;
}
size_t file_size = file_info->mSize;
hdfsFreeFileInfo(file_info, 1);
if ((file_size % kSlotSizeBytes) != 0) {
throw CorruptPersistentStorage();
}
return file_size / kSlotSizeBytes;
}
FileStatus Hdfs::getFileStatus(string path)
{
hdfsFileInfo* fi = hdfsGetPathInfo(_getFs(), path.data());
if (fi == NULL)
{
throw ios_base::failure("Error retrieving file status. (" + path + ")");
}
FileStatus result;
result._setAccessTime(fi->mLastAccess);
result._setBlockSize(fi->mBlockSize);
result._setModificationTime(fi->mLastMod);
result._setLen(fi->mSize);
result._setKind(fi->mKind);
result._setPath(fi->mName);
hdfsFreeFileInfo(fi, 1);
return result;
}
int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos)
{
struct webhdfsFileHandle *wf;
hdfsFileInfo *fileInfo = NULL;
int ret = 0;
if (!fs || !file || (file->type == OUTPUT) || (desiredPos < 0)) {
ret = EINVAL;
goto done;
}
wf = file->file;
if (!wf) {
ret = EINVAL;
goto done;
}
fileInfo = hdfsGetPathInfo(fs, wf->absPath);
if (!fileInfo) {
ret = errno;
goto done;
}
if (desiredPos > fileInfo->mSize) {
fprintf(stderr,
"hdfsSeek for %s failed since the desired position %" PRId64
" is beyond the size of the file %" PRId64 "\n",
wf->absPath, desiredPos, fileInfo->mSize);
ret = ENOTSUP;
goto done;
}
file->offset = desiredPos;
done:
if (fileInfo) {
hdfsFreeFileInfo(fileInfo, 1);
}
if (ret) {
errno = ret;
return -1;
}
return 0;
}
bs_file_info_t *
hdfs_file_info(struct back_storage *storage,const char* path){
//HLOG_DEBUG("hdfs -- enter func %s", __func__);
char full_path[256];
build_hdfs_path(full_path,storage->dir,storage->fs_name,path);
hdfsFileInfo *hinfo = hdfsGetPathInfo((hdfsFS)storage->fs_handler,full_path);
if(NULL == hinfo){
//HLOG_ERROR("hdfsGetPathInfo error");
return NULL;
}
bs_file_info_t *info = (bs_file_info_t*)g_malloc0(sizeof(bs_file_info_t));
if (NULL == info) {
//HLOG_ERROR("Allocate Error!");
return NULL;
}
strcpy((char *) info->name, (const char *) hinfo->mName);
info->is_dir = 0;//?
info->size = hinfo->mSize;
info->lmtime = hinfo->mLastMod;
free(hinfo);
//HLOG_DEBUG("hdfs -- leave func %s", __func__);
return info;
}
long libhdfsconnector::getFileSize(const char * filename)
{
if (!fs)
{
fprintf(stderr, "Could not connect to hdfs");
return RETURN_FAILURE;
}
hdfsFileInfo *fileInfo = NULL;
if ((fileInfo = hdfsGetPathInfo(fs, filename)) != NULL)
{
long fsize = fileInfo->mSize;
hdfsFreeFileInfo(fileInfo, 1);
return fsize;
}
else
{
fprintf(stderr, "Error: hdfsGetPathInfo for %s - FAILED!\n", filename);
return RETURN_FAILURE;
}
return RETURN_FAILURE;
}
/**
* hdfsCopy - Copy file from one filesystem to another.
*
* @param srcFS The handle to source filesystem.
* @param src The path of source file.
* @param dstFS The handle to destination filesystem.
* @param dst The path of destination file.
* @return Returns 0 on success, -1 on error.
*/
int
hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst)
{
char *block = NULL, *src_abs, *dst_abs;
hdfsFileInfo *srcinfo = NULL;
int res = -1;
hdfsFile a = NULL, b = NULL;
tOffset twritten = 0;
src_abs = _makeabs(srcFS, src);
dst_abs = _makeabs(dstFS, dst);
if (hdfsExists(srcFS, src_abs) == -1) {
ERR(ENOENT, "'%s' doesn't exist on srcFS", src_abs);
goto out;
}
srcinfo = hdfsGetPathInfo(srcFS, src_abs);
if (!srcinfo) {
ERR(errno, "hdfsGetPathInfo failed");
goto out;
}
if (srcinfo->mKind == kObjectKindDirectory) {
ERR(ENOTSUP, "hdfsCopy can't do directories right now");
goto out;
}
a = hdfsOpenFile(srcFS, src_abs, O_RDONLY, 0, 0, 0);
if (!a) {
ERR(errno, "hdfsOpenFile failed");
goto out;
}
b = hdfsOpenFile(dstFS, dst_abs, O_WRONLY, 0, DEFAULT_REPLICATION,
DEFAULT_BLOCK_SIZE);
if (!b) {
ERR(errno, "hdfsOpenFile failed");
goto out;
}
block = malloc(DEFAULT_BLOCK_SIZE);
assert(block);
while (twritten < srcinfo->mSize) {
tSize toread, read, written;
toread = _imin(DEFAULT_BLOCK_SIZE, srcinfo->mSize - twritten);
read = hdfsRead(srcFS, a, block, toread);
if (read == -1) {
ERR(errno, "hdfsRead failed");
goto out;
}
written = hdfsWrite(dstFS, b, block, read);
if (written == -1) {
ERR(errno, "hdfsWrite failed");
goto out;
}
assert(written == read);
twritten += written;
}
res = 0;
out:
if (a)
hdfsCloseFile(srcFS, a);
if (b)
hdfsCloseFile(dstFS, b);
if (src_abs != src)
free(src_abs);
if (dst_abs != dst)
free(dst_abs);
if (block)
free(block);
if (srcinfo)
hdfsFreeFileInfo(srcinfo, 1);
return res;
}
static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
const struct tlhPaths *paths)
{
char tmp[4096];
hdfsFile file;
int ret, expected, numEntries;
hdfsFileInfo *fileInfo;
struct hdfsReadStatistics *readStats = NULL;
if (hdfsExists(fs, paths->prefix) == 0) {
EXPECT_ZERO(hdfsDelete(fs, paths->prefix, 1));
}
EXPECT_ZERO(hdfsCreateDirectory(fs, paths->prefix));
EXPECT_ZERO(doTestGetDefaultBlockSize(fs, paths->prefix));
/* There should be no entry in the directory. */
errno = EACCES; // see if errno is set to 0 on success
EXPECT_NULL_WITH_ERRNO(hdfsListDirectory(fs, paths->prefix, &numEntries), 0);
if (numEntries != 0) {
fprintf(stderr, "hdfsListDirectory set numEntries to "
"%d on empty directory.", numEntries);
}
/* There should not be any file to open for reading. */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0));
/* hdfsOpenFile should not accept mode = 3 */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, 3, 0, 0, 0));
file = hdfsOpenFile(fs, paths->file1, O_WRONLY, 0, 0, 0);
EXPECT_NONNULL(file);
/* TODO: implement writeFully and use it here */
expected = (int)strlen(paths->prefix);
ret = hdfsWrite(fs, file, paths->prefix, expected);
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
"it wrote %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(hdfsFlush(fs, file));
EXPECT_ZERO(hdfsHSync(fs, file));
EXPECT_ZERO(hdfsCloseFile(fs, file));
/* There should be 1 entry in the directory. */
EXPECT_NONNULL(hdfsListDirectory(fs, paths->prefix, &numEntries));
if (numEntries != 1) {
fprintf(stderr, "hdfsListDirectory set numEntries to "
"%d on directory containing 1 file.", numEntries);
}
/* Let's re-open the file for reading */
file = hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0);
EXPECT_NONNULL(file);
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
errno = 0;
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalBytesRead);
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalLocalBytesRead);
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalShortCircuitBytesRead);
hdfsFileFreeReadStatistics(readStats);
/* TODO: implement readFully and use it here */
ret = hdfsRead(fs, file, tmp, sizeof(tmp));
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
"it read %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
errno = 0;
EXPECT_UINT64_EQ((uint64_t)expected, readStats->totalBytesRead);
hdfsFileFreeReadStatistics(readStats);
EXPECT_ZERO(hdfsFileClearReadStatistics(file));
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
EXPECT_UINT64_EQ((uint64_t)0, readStats->totalBytesRead);
hdfsFileFreeReadStatistics(readStats);
EXPECT_ZERO(memcmp(paths->prefix, tmp, expected));
EXPECT_ZERO(hdfsCloseFile(fs, file));
// TODO: Non-recursive delete should fail?
//EXPECT_NONZERO(hdfsDelete(fs, prefix, 0));
EXPECT_ZERO(hdfsCopy(fs, paths->file1, fs, paths->file2));
EXPECT_ZERO(hdfsChown(fs, paths->file2, NULL, NULL));
EXPECT_ZERO(hdfsChown(fs, paths->file2, NULL, "doop"));
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("doop", fileInfo->mGroup));
EXPECT_ZERO(hdfsFileIsEncrypted(fileInfo));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha", "doop2"));
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha2", NULL));
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha2", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
snprintf(tmp, sizeof(tmp), "%s/nonexistent-file-name", paths->prefix);
EXPECT_NEGATIVE_ONE_WITH_ERRNO(hdfsChown(fs, tmp, "ha3", NULL), ENOENT);
return 0;
}
int dfs_open(const char *path, struct fuse_file_info *fi)
{
hdfsFS fs = NULL;
dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
dfs_fh *fh = NULL;
int mutexInit = 0, ret;
TRACE1("open", path)
// check params and the context var
assert(path);
assert('/' == *path);
assert(dfs);
// 0x8000 is always passed in and hadoop doesn't like it, so killing it here
// bugbug figure out what this flag is and report problem to Hadoop JIRA
int flags = (fi->flags & 0x7FFF);
// retrieve dfs specific data
fh = (dfs_fh*)calloc(1, sizeof (dfs_fh));
if (!fh) {
ERROR("Malloc of new file handle failed");
ret = -EIO;
goto error;
}
ret = fuseConnectAsThreadUid(&fh->conn);
if (ret) {
fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs "
"connection! error %d.\n", ret);
ret = -EIO;
goto error;
}
fs = hdfsConnGetFs(fh->conn);
if (flags & O_RDWR) {
hdfsFileInfo *info = hdfsGetPathInfo(fs, path);
if (info == NULL) {
// File does not exist (maybe?); interpret it as a O_WRONLY
// If the actual error was something else, we'll get it again when
// we try to open the file.
flags ^= O_RDWR;
flags |= O_WRONLY;
} else {
// File exists; open this as read only.
flags ^= O_RDWR;
flags |= O_RDONLY;
}
}
if ((fh->hdfsFH = hdfsOpenFile(fs, path, flags, 0, 0, 0)) == NULL) {
ERROR("Could not open file %s (errno=%d)", path, errno);
if (errno == 0 || errno == EINTERNAL) {
ret = -EIO;
goto error;
}
ret = -errno;
goto error;
}
ret = pthread_mutex_init(&fh->mutex, NULL);
if (ret) {
fprintf(stderr, "dfs_open: error initializing mutex: error %d\n", ret);
ret = -EIO;
goto error;
}
mutexInit = 1;
if (fi->flags & O_WRONLY || fi->flags & O_CREAT) {
fh->buf = NULL;
} else {
assert(dfs->rdbuffer_size > 0);
fh->buf = (char*)malloc(dfs->rdbuffer_size * sizeof(char));
if (NULL == fh->buf) {
ERROR("Could not allocate memory for a read for file %s\n", path);
ret = -EIO;
goto error;
}
fh->buffersStartOffset = 0;
fh->bufferSize = 0;
}
fi->fh = (uint64_t)fh;
return 0;
error:
if (fh) {
if (mutexInit) {
pthread_mutex_destroy(&fh->mutex);
}
free(fh->buf);
if (fh->hdfsFH) {
hdfsCloseFile(fs, fh->hdfsFH);
}
if (fh->conn) {
hdfsConnRelease(fh->conn);
}
free(fh);
}
return ret;
}
void dir_cpi_impl::sync_copy (saga::impl::void_t & ret,
saga::url src,
saga::url dst,
int flags)
{
instance_data idata (this);
saga::url url(idata->location_);
// handle the files
boost::filesystem::path src_location (idata->location_.get_path(), boost::filesystem::native);
boost::filesystem::path dst_location (src_location);
// complete paths
boost::filesystem::path src_path (src.get_path(), boost::filesystem::native);
boost::filesystem::path dest_path (dst.get_path(), boost::filesystem::native);
if ( ! src_path.has_root_path () )
src_location /= src_path;
else
src_location = src_path;
if ( ! dest_path.has_root_path () )
dst_location /= dest_path;
else
dst_location = dest_path;
bool is_src_dir = false;
if(hdfsExists(fs_, src_location.string().c_str()) == 0)
{
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, src_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_src_dir = true;
hdfsFreeFileInfo(info, 1);
}
// src location refers to a is a directory
if (is_src_dir) {
SAGA_ADAPTOR_THROW("Cannot copy directory at moment.", saga::NotImplemented);
}
else {
//Check to see if dst_location is a directory
bool is_dst_dir = false;
bool dst_exists = false;
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
dst_exists = true;
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, dst_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_dst_dir = true;
hdfsFreeFileInfo(info, 1);
}
else
{
SAGA_ADAPTOR_THROW("Path does not exists!", saga::NoSuccess);
}
if (is_dst_dir)
dst_location /= src_location.leaf();
// remove the file/directory if it exists and we should overwrite
if ((flags & saga::name_space::Overwrite) && dst_exists) {
/* if (is_dst_dir)
fs::remove_all(dst_location);*/
saga_hdfs_delete(fs_, dst_location.string().c_str());
}
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
SAGA_OSSTREAM strm;
strm << "namespace_dir_cpi_impl<Base>::sync_copy: "
"target file already exists: " << dst.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::AlreadyExists);
}
if(hdfsCopy(fs_, src_location.string().c_str(), fs_, dst_location.string().c_str()) != 0)
{
SAGA_OSSTREAM strm;
strm << "namespace_dir_cpi_impl<Base>::sync_copy: "
"target file did not copy successfully: " << dst.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::NoSuccess);
}
}
}
void dir_cpi_impl::sync_move (saga::impl::void_t & ret,
saga::url src,
saga::url dest,
int flags)
{
instance_data idata (this);
// verify current working directory is local
saga::url url(idata->location_);
// handle the files
boost::filesystem::path src_location (idata->location_.get_path(), boost::filesystem::native);
boost::filesystem::path dst_location (src_location);
// complete paths
boost::filesystem::path src_path (src.get_path(), boost::filesystem::native);
boost::filesystem::path dest_path (dest.get_path(), boost::filesystem::native);
if ( ! src_path.has_root_path () )
src_location /= src_path;
else
src_location = src_path;
if ( ! dest_path.has_root_path () )
dst_location /= dest_path;
else
dst_location = dest_path;
bool is_src_dir;
bool is_dst_dir;
if(hdfsExists(fs_, src_location.string().c_str()) == 0)
{
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, src_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_src_dir = true;
else
is_src_dir = false;
hdfsFreeFileInfo(info, 1);
}
bool dst_exists = false;
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
dst_exists = true;
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, dst_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_dst_dir = true;
else
is_dst_dir = false;
hdfsFreeFileInfo(info, 1);
}
if (!is_src_dir && is_dst_dir)
dst_location /= src_location.leaf();
if ((flags & saga::name_space::Overwrite) && dst_exists) {
/* if (is_dst_dir)
fs::remove_all(dst_location);*/
saga_hdfs_delete(fs_, dst_location.string().c_str());
}
// if destination still exists raise an error
dst_exists = false;
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
SAGA_OSSTREAM strm;
strm << "namespace_dir_cpi_impl<Base>::sync_move: "
"target file already exists: " << dest.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::AlreadyExists);
}
if(hdfsMove(fs_, src_location.string().c_str(), fs_, dst_location.string().c_str()) != 0)
{
SAGA_ADAPTOR_THROW("Unable to move files", saga::NoSuccess);
}
}
int main(int argc, char **argv) {
hdfsFS fs = hdfsConnect("default", 0);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
hdfsFS lfs = hdfsConnect(NULL, 0);
if(!lfs) {
fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
exit(-1);
}
const char* writePath = "/tmp/testfile.txt";
{
//Write tests
hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
char* buffer = "Hello, World!";
tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
tOffset currentPos = -1;
if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
fprintf(stderr,
"Failed to get current file position correctly! Got %ld!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %ld\n", currentPos);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
}
{
//Read tests
const char* readPath = "/tmp/testfile.txt";
int exists = hdfsExists(fs, readPath);
if (exists) {
fprintf(stderr, "Failed to validate existence of %s\n", readPath);
exit(-1);
}
hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", readPath);
exit(-1);
}
fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));
tOffset seekPos = 1;
if(hdfsSeek(fs, readFile, seekPos)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
tOffset currentPos = -1;
if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
fprintf(stderr,
"Failed to get current file position correctly! Got %ld!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %ld\n", currentPos);
static char buffer[32];
tSize num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
hdfsCloseFile(fs, readFile);
}
int totalResult = 0;
int result = 0;
{
//Generic file-system operations
const char* srcPath = "/tmp/testfile.txt";
const char* dstPath = "/tmp/testfile2.txt";
fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
const char* slashTmp = "/tmp";
const char* newDirectory = "/tmp/newdir";
fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) ? "Failed!" : "Success!"));
totalResult += result;
char buffer[256];
const char *resp;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsGetDefaultBlockSize: %ld\n", hdfsGetDefaultBlockSize(fs));
fprintf(stderr, "hdfsGetCapacity: %ld\n", hdfsGetCapacity(fs));
fprintf(stderr, "hdfsGetUsed: %ld\n", hdfsGetUsed(fs));
hdfsFileInfo *fileInfo = NULL;
if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
fprintf(stderr, "Name: %s, ", fileInfo->mName);
fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
fprintf(stderr, "BlockSize: %ld, ", fileInfo->mBlockSize);
fprintf(stderr, "Size: %ld, ", fileInfo->mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
char permissions[10];
permission_disp(fileInfo->mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
hdfsFreeFileInfo(fileInfo, 1);
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
}
hdfsFileInfo *fileList = 0;
int numEntries = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
int i = 0;
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
fprintf(stderr, "BlockSize: %ld, ", fileList[i].mBlockSize);
fprintf(stderr, "Size: %ld, ", fileList[i].mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
char permissions[10];
permission_disp(fileList[i].mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
}
hdfsFreeFileInfo(fileList, numEntries);
} else {
if (errno) {
totalResult++;
fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
} else {
fprintf(stderr, "Empty directory!\n");
}
}
char*** hosts = hdfsGetHosts(fs, srcPath, 0, 1);
if(hosts) {
fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
int i=0;
while(hosts[i]) {
int j = 0;
while(hosts[i][j]) {
fprintf(stderr,
"\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
++j;
}
++i;
}
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
}
char *newOwner = "root";
// setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
short newPerm = 0666;
// chown write
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) ? "Failed!" : "Success!"));
totalResult += result;
// chmod write
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) ? "Failed!" : "Success!"));
totalResult += result;
sleep(2);
tTime newMtime = time(NULL);
tTime newAtime = time(NULL);
// utime write
fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) ? "Failed!" : "Success!"));
totalResult += result;
// chown/chmod/utime read
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!"));
totalResult += result;
// will later use /tmp/ as a different user so enable it
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr,"newMTime=%ld\n",newMtime);
fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) ? "Failed!" : "Success!"));
totalResult += result;
// No easy way to turn on access times from hdfs_test right now
// fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) ? "Failed!" : "Success!"));
// totalResult += result;
hdfsFreeFileInfo(finfo, 1);
// Clean up
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
}
totalResult += (hdfsDisconnect(fs) != 0);
{
//
// Now test as connecting as a specific user
// This is only meant to test that we connected as that user, not to test
// the actual fs user capabilities. Thus just create a file and read
// the owner is correct.
const char *tuser = "******";
const char* writePath = "/tmp/usertestfile.txt";
const char **groups = (const char**)malloc(sizeof(char*)* 2);
groups[0] = "users";
groups[1] = "nobody";
fs = hdfsConnectAsUser("default", 0, tuser, groups, 2);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
exit(-1);
}
hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
char* buffer = "Hello, World!";
tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!"));
totalResult += result;
}
totalResult += (hdfsDisconnect(fs) != 0);
if (totalResult != 0) {
return -1;
} else {
return 0;
}
}
static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs)
{
char prefix[256], tmp[256];
hdfsFile file;
int ret, expected;
hdfsFileInfo *fileInfo;
snprintf(prefix, sizeof(prefix), "/tlhData%04d", ti->threadIdx);
if (hdfsExists(fs, prefix) == 0) {
EXPECT_ZERO(hdfsDelete(fs, prefix, 1));
}
EXPECT_ZERO(hdfsCreateDirectory(fs, prefix));
snprintf(tmp, sizeof(tmp), "%s/file", prefix);
EXPECT_NONNULL(hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0));
file = hdfsOpenFile(fs, tmp, O_WRONLY, 0, 0, 0);
EXPECT_NONNULL(file);
/* TODO: implement writeFully and use it here */
expected = (int)strlen(prefix);
ret = hdfsWrite(fs, file, prefix, expected);
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
"it wrote %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(hdfsFlush(fs, file));
EXPECT_ZERO(hdfsCloseFile(fs, file));
/* Let's re-open the file for reading */
file = hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0);
EXPECT_NONNULL(file);
/* TODO: implement readFully and use it here */
ret = hdfsRead(fs, file, tmp, sizeof(tmp));
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
"it read %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(memcmp(prefix, tmp, expected));
EXPECT_ZERO(hdfsCloseFile(fs, file));
snprintf(tmp, sizeof(tmp), "%s/file", prefix);
EXPECT_NONZERO(hdfsChown(fs, tmp, NULL, NULL));
EXPECT_ZERO(hdfsChown(fs, tmp, NULL, "doop"));
fileInfo = hdfsGetPathInfo(fs, tmp);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("doop", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, tmp, "ha", "doop2"));
fileInfo = hdfsGetPathInfo(fs, tmp);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, tmp, "ha2", NULL));
fileInfo = hdfsGetPathInfo(fs, tmp);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha2", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsDelete(fs, prefix, 1));
return 0;
}
