tSize HDFSReadWrite::HDFSWrite(std::string fileName, const char* buffer,
tSize len) {
fileName = "/" + fileName;
int flag = O_WRONLY | O_CREAT;
if (hdfsExists(::HdfsConnectionPool::hdfs(), fileName.c_str()) == 0) {
flag = O_WRONLY | O_APPEND;
}
hdfsFile writeFile = hdfsOpenFile(::HdfsConnectionPool::hdfs(),
fileName.c_str(), flag, 0, 0, 0);
if (writeFile != NULL) {
tSize bytesWriten = hdfsWrite(::HdfsConnectionPool::hdfs(), writeFile,
buffer, len);
;
if (hdfsFlush(::HdfsConnectionPool::hdfs(), writeFile) == -1) {
return -1;
}
hdfsCloseFile(::HdfsConnectionPool::hdfs(), writeFile);
return bytesWriten;
} else {
DEBUG("hdfs open file failed!");
}
return -1;
}
HdfsFile::HdfsFile(const char* fname, const char* mode, unsigned /* opts */) :
IDBDataFile( fname ),
m_file(0),
m_fs(0)
{
int savedErrno;
m_flags = modeStrToFlags(mode);
if( m_flags == -1 )
{
ostringstream oss;
oss << "Error opening file " << fname << " - unsupported mode " << mode;
throw std::runtime_error(oss.str());
}
m_fs = HdfsFsCache::fs();
// @bug5476, HDFS do not support O_CREAT|O_APPEND as of 2.0
// special handle for O_APPEND
if ((m_flags & O_APPEND) && (hdfsExists(m_fs, fname) != 0))
m_flags &= ~O_APPEND;
m_file = hdfsOpenFile(m_fs, fname, m_flags, 0, 0, 0);
savedErrno = errno;
if(!m_file)
{
ostringstream oss;
oss << "Error opening file " << fname << ": " << strerror(savedErrno);
throw std::runtime_error(oss.str());
}
}
void dir_cpi_impl::sync_is_dir (bool & is_dir,
saga::url url)
{
instance_data idata (this);
is_dir = false;
saga::url dir_url(idata->location_);
boost::filesystem::path name (url.get_path(), boost::filesystem::native);
boost::filesystem::path path (idata->location_.get_path(), boost::filesystem::native);
if ( ! name.has_root_path () )
path /= name;
else
path = name;
if(hdfsExists(fs_, path.string().c_str()) == 0)
{
//Check to see if it is a directory
hdfsFileInfo *info;
instance_data idata(this);
info = hdfsGetPathInfo(fs_, path.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_dir = true;
hdfsFreeFileInfo(info, 1);
}
}
bool MaprFileSystem::Exists(const std::string& uri){
std::string path = GetUriPathOrDie(uri);
std::string host = "default";
hdfsFS fs = hdfsConnect(host.c_str(), 0); // use default config file settings
CHECK(fs) << "Can't connect to filesystem for this uri: " << uri;
bool exists = (hdfsExists(fs, path.c_str()) == 0);
CHECK_EQ(hdfsDisconnect(fs), 0);
return exists;
}
bool HdfsConnector::assgin_open_file(open_flag open_flag_){
vector<vector<string> >::iterator prj_writepath;
vector<string>::iterator par_writepath;
for (prj_writepath = writepath.begin(); prj_writepath != writepath.end(); prj_writepath++)
{
vector<hdfsFile> prj_writefile;
prj_writefile.clear();
for (par_writepath = (*prj_writepath).begin(); par_writepath != (*prj_writepath).end(); par_writepath++)
{
switch (open_flag_)
{
case CREATEE:
{
if (hdfsExists(fs, (*par_writepath).c_str()) == 0)
cout << "[WARNINIG: Hdfsconnector.cpp->assgin_open_file()]: The file " << *par_writepath << " is already exits! It will be override!\n";
prj_writefile.push_back(hdfsOpenFile(fs, (*par_writepath).c_str(), O_WRONLY|O_CREAT, 0, 0, 0));
break;
}
case APPENDD:
{
if (hdfsExists(fs, (*par_writepath).c_str()) == -1)
{
prj_writefile.push_back(hdfsOpenFile(fs, (*par_writepath).c_str(), O_WRONLY|O_CREAT, 0, 0, 0));
break;
// cout << "[ERROR: Hdfsconnector.cpp->assgin_open_file()]: The file " << *par_writepath << "is not exits!\n";
// return false;
}
prj_writefile.push_back(hdfsOpenFile(fs, (*par_writepath).c_str(), O_WRONLY|O_APPEND, 0, 0, 0));
break;
}
default:
{
cout << "[ERROR: Hdfsconnector.cpp->assgin_open_file()]: Illegal file open flag for data loading!\n";
return false;
}
}
}
file_handles_.push_back(prj_writefile);
}
return true;
}
int hdfs_file_is_exist(struct back_storage *storage, const char *path){
//HLOG_DEBUG("hdfs -- enter func %s", __func__);
char full_path[256];
build_hdfs_path(full_path, storage->dir, storage->fs_name, path);
//HLOG_DEBUG("hdfs full path %s", full_path);
if (0 != hdfsExists((hdfsFS)storage->fs_handler, full_path)) {
//HLOG_ERROR("hdfsExists error");
return -1;
}
//HLOG_DEBUG("hdfs -- leave func %s", __func__);
return 0;
}
void dir_cpi_impl::sync_remove (saga::impl::void_t & ret,
saga::url url,
int flags)
{
instance_data idata (this);
saga::url dir_url(idata->location_);
boost::filesystem::path src_location (idata->location_.get_path(), boost::filesystem::native);
// complete paths
boost::filesystem::path src_path (url.get_path(), boost::filesystem::native);
if ( ! src_path.has_root_path () )
src_location /= src_path;
else
src_location = src_path;
bool is_src_dir = false;
if(hdfsExists(fs_, src_location.string().c_str()) != 0)
{
SAGA_ADAPTOR_THROW("directory::remove: Can't remove directory: "
"Does not exist", saga::DoesNotExist);
}
else
{
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, src_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_src_dir = true;
else
is_src_dir = false;
hdfsFreeFileInfo(info, 1);
}
if (is_src_dir) {
if (saga::name_space::Recursive != flags)
{
SAGA_ADAPTOR_THROW("directory::remove: Can't remove directory. "
"Please use recursive mode!", saga::BadParameter);
}
else {
saga_hdfs_delete(fs_, src_location.string().c_str());
}
}
else {
saga_hdfs_delete(fs_, src_location.string().c_str());
}
}
int hdfs_file_mkdir(struct back_storage *storage, const char *dir_path){
//HLOG_DEBUG("hdfs -- enter func %s", __func__);
char full_path[256];
build_hdfs_path(full_path, storage->dir, storage->fs_name, dir_path);
//HLOG_DEBUG("hdfs mkdir:%s",full_path);
if (0 == hdfsExists((hdfsFS)storage->fs_handler, full_path)) {
return -1;
}
if (0 != hdfsCreateDirectory((hdfsFS)storage->fs_handler, full_path)) {
//HLOG_DEBUG("hdfsCreateDirectory error");
return -1;
}
//HLOG_DEBUG("hdfs -- leave func %s", __func__);
return 0;
}
/**
* hdfsSetWorkingDirectory - Set the working directory. All relative
* paths will be resolved relative to it.
* @param fs The configured filesystem handle.
* @param path The path of the new 'cwd'.
* @return Returns 0 on success, -1 on error.
*/
int
hdfsSetWorkingDirectory(hdfsFS fs, const char* path)
{
char *path_abs = _makeabs(fs, path);
struct hdfsFS_internal *client = fs;
int res;
res = hdfsExists(fs, path_abs);
if (res != -1) {
free(client->fs_cwd);
client->fs_cwd = strdup(path_abs);
assert(client->fs_cwd);
}
if (path_abs != path)
free(path_abs);
return res;
}
static PyObject *
pyhdfsFS_delete(char* filepath)
{
// Parse HDFS information
char* hostport = (char *)malloc(strlen(filepath)+1);
char* buf = NULL;
char* path;
char* portStr;
char* host;
int hdfsPort;
int ret = 0;
ret = sscanf(filepath, "hdfs://%s", hostport);
host = strtok_r(hostport, ":", &buf);
portStr = strtok_r(NULL, "/", &buf);
ret = sscanf(portStr, "%d", &hdfsPort);
hdfsFS fs = hdfsConnect(host, hdfsPort);
if (!fs)
{
PyErr_SetString(exception, "Cannot connect to host");
return exception;
}
path = (char*) malloc(strlen(buf)+2);
path[0] = '/';
memcpy(path+1,buf,strlen(buf));
path[strlen(buf)+1] = '\0';
if (hdfsExists(fs, path) == 0)
{
hdfsDelete(fs, path);
}
else
{
PyErr_SetString(exception, "Cannot delete file");
return NULL;
}
free(hostport);
free(path);
return Py_BuildValue("i",1);
}
void dir_cpi_impl::sync_exists (bool & exists,
saga::url url)
{
instance_data idata (this);
saga::url dir_url(idata->location_);
boost::filesystem::path name (url.get_path(), boost::filesystem::native);
boost::filesystem::path path (idata->location_.get_path(), boost::filesystem::native);
if ( ! name.has_root_path () )
path /= name;
else
path = name;
exists = false;
if(hdfsExists(fs_, path.string().c_str()) == 0)
{
exists = true;
}
}
static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
const struct tlhPaths *paths)
{
char tmp[4096];
hdfsFile file;
int ret, expected, numEntries;
hdfsFileInfo *fileInfo;
struct hdfsReadStatistics *readStats = NULL;
if (hdfsExists(fs, paths->prefix) == 0) {
EXPECT_ZERO(hdfsDelete(fs, paths->prefix, 1));
}
EXPECT_ZERO(hdfsCreateDirectory(fs, paths->prefix));
EXPECT_ZERO(doTestGetDefaultBlockSize(fs, paths->prefix));
/* There should be no entry in the directory. */
errno = EACCES; // see if errno is set to 0 on success
EXPECT_NULL_WITH_ERRNO(hdfsListDirectory(fs, paths->prefix, &numEntries), 0);
if (numEntries != 0) {
fprintf(stderr, "hdfsListDirectory set numEntries to "
"%d on empty directory.", numEntries);
}
/* There should not be any file to open for reading. */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0));
/* hdfsOpenFile should not accept mode = 3 */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, 3, 0, 0, 0));
file = hdfsOpenFile(fs, paths->file1, O_WRONLY, 0, 0, 0);
EXPECT_NONNULL(file);
/* TODO: implement writeFully and use it here */
expected = (int)strlen(paths->prefix);
ret = hdfsWrite(fs, file, paths->prefix, expected);
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
"it wrote %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(hdfsFlush(fs, file));
EXPECT_ZERO(hdfsHSync(fs, file));
EXPECT_ZERO(hdfsCloseFile(fs, file));
/* There should be 1 entry in the directory. */
EXPECT_NONNULL(hdfsListDirectory(fs, paths->prefix, &numEntries));
if (numEntries != 1) {
fprintf(stderr, "hdfsListDirectory set numEntries to "
"%d on directory containing 1 file.", numEntries);
}
/* Let's re-open the file for reading */
file = hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0);
EXPECT_NONNULL(file);
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
errno = 0;
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalBytesRead);
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalLocalBytesRead);
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalShortCircuitBytesRead);
hdfsFileFreeReadStatistics(readStats);
/* TODO: implement readFully and use it here */
ret = hdfsRead(fs, file, tmp, sizeof(tmp));
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
"it read %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
errno = 0;
EXPECT_UINT64_EQ((uint64_t)expected, readStats->totalBytesRead);
hdfsFileFreeReadStatistics(readStats);
EXPECT_ZERO(hdfsFileClearReadStatistics(file));
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
EXPECT_UINT64_EQ((uint64_t)0, readStats->totalBytesRead);
hdfsFileFreeReadStatistics(readStats);
EXPECT_ZERO(memcmp(paths->prefix, tmp, expected));
EXPECT_ZERO(hdfsCloseFile(fs, file));
// TODO: Non-recursive delete should fail?
//EXPECT_NONZERO(hdfsDelete(fs, prefix, 0));
EXPECT_ZERO(hdfsCopy(fs, paths->file1, fs, paths->file2));
EXPECT_ZERO(hdfsChown(fs, paths->file2, NULL, NULL));
EXPECT_ZERO(hdfsChown(fs, paths->file2, NULL, "doop"));
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("doop", fileInfo->mGroup));
EXPECT_ZERO(hdfsFileIsEncrypted(fileInfo));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha", "doop2"));
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha2", NULL));
fileInfo = hdfsGetPathInfo(fs, paths->file2);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha2", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
snprintf(tmp, sizeof(tmp), "%s/nonexistent-file-name", paths->prefix);
EXPECT_NEGATIVE_ONE_WITH_ERRNO(hdfsChown(fs, tmp, "ha3", NULL), ENOENT);
return 0;
}
int libhdfsconnector::mergeFile()
{
if (nodeID == 0)
{
if (!fs)
{
fprintf(stderr, "Could not connect to hdfs on");
return RETURN_FAILURE;
}
fprintf(stderr, "merging %d file(s) into %s\n", clusterCount, fileName);
fprintf(stderr, "Opening %s for writing!\n", fileName);
hdfsFile writeFile = hdfsOpenFile(fs, fileName, O_CREAT | O_WRONLY, 0, filereplication, 0);
if (!writeFile)
{
fprintf(stderr, "Failed to open %s for writing!\n", fileName);
return EXIT_FAILURE;
}
tSize totalBytesWritten = 0;
for (unsigned node = 0; node < clusterCount; node++)
{
if (node > 0)
{
writeFile = hdfsOpenFile(fs, fileName, O_WRONLY | O_APPEND, 0, filereplication, 0);
fprintf(stderr, "Re-opening %s for append!\n", fileName);
}
unsigned bytesWrittenSinceLastFlush = 0;
string filepartname;
createFilePartName(&filepartname, fileName, node, clusterCount);
if (hdfsExists(fs, filepartname.c_str()) == 0)
{
fprintf(stderr, "Opening readfile %s\n", filepartname.c_str());
hdfsFile readFile = hdfsOpenFile(fs, filepartname.c_str(), O_RDONLY, 0, 0, 0);
if (!readFile)
{
fprintf(stderr, "Failed to open %s for reading!\n", fileName);
return EXIT_FAILURE;
}
unsigned char buffer[bufferSize + 1];
while (hdfsAvailable(fs, readFile))
{
tSize num_read_bytes = hdfsRead(fs, readFile, buffer, bufferSize);
if (num_read_bytes <= 0)
break;
tSize bytesWritten = 0;
try
{
bytesWritten = hdfsWrite(fs, writeFile, (void*) buffer, num_read_bytes);
totalBytesWritten += bytesWritten;
bytesWrittenSinceLastFlush += bytesWritten;
if (bytesWrittenSinceLastFlush >= flushThreshold)
{
if (hdfsFlush(fs, writeFile))
{
fprintf(stderr, "Failed to 'flush' %s\n", fileName);
return EXIT_FAILURE;
}
bytesWrittenSinceLastFlush = 0;
}
} catch (...)
{
fprintf(stderr, "Issue detected during HDFSWrite\n");
fprintf(stderr, "Bytes written in current iteration: %d\n", bytesWritten);
return EXIT_FAILURE;
}
}
if (hdfsFlush(fs, writeFile))
{
fprintf(stderr, "Failed to 'flush' %s\n", fileName);
return EXIT_FAILURE;
}
fprintf(stderr, "Closing readfile %s\n", filepartname.c_str());
hdfsCloseFile(fs, readFile);
if (cleanmerge)
{
#ifdef HADOOP_GT_21
hdfsDelete(fs, filepartname.c_str(), 0);
#else
hdfsDelete(fs, filepartname.c_str());
#endif
}
}
else
{
fprintf(stderr, "Could not merge, part %s was not located\n", filepartname.c_str());
return EXIT_FAILURE;
}
fprintf(stderr, "Closing writefile %s\n", fileName);
if (hdfsCloseFile(fs, writeFile) != 0)
fprintf(stderr, "Could not close writefile %s\n", fileName);
}
if (cleanmerge)
{
string filecontainer;
filecontainer.assign(fileName);
filecontainer.append("-parts");
#ifdef HADOOP_GT_21
hdfsDelete(fs, filecontainer.c_str(), 0);
#else
hdfsDelete(fs, filecontainer.c_str());
#endif
}
}
return EXIT_SUCCESS;
}
/**
* call-seq:
* hdfs.exist?(path) -> file_existence
*
* Checks if a file exists at the supplied path. If file exists, returns True;
* if not, returns False.
*/
VALUE HDFS_File_System_exist(VALUE self, VALUE path) {
FSData* data = get_FSData(self);
int success = hdfsExists(data->fs, StringValuePtr(path));
return success == 0 ? Qtrue : Qfalse;
}
bool HdfsFileSystem::exists(const char *pathname) const
{
int ret = hdfsExists(m_fs,pathname);
return ret == 0;
}
/**
* hdfsCopy - Copy file from one filesystem to another.
*
* @param srcFS The handle to source filesystem.
* @param src The path of source file.
* @param dstFS The handle to destination filesystem.
* @param dst The path of destination file.
* @return Returns 0 on success, -1 on error.
*/
int
hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst)
{
char *block = NULL, *src_abs, *dst_abs;
hdfsFileInfo *srcinfo = NULL;
int res = -1;
hdfsFile a = NULL, b = NULL;
tOffset twritten = 0;
src_abs = _makeabs(srcFS, src);
dst_abs = _makeabs(dstFS, dst);
if (hdfsExists(srcFS, src_abs) == -1) {
ERR(ENOENT, "'%s' doesn't exist on srcFS", src_abs);
goto out;
}
srcinfo = hdfsGetPathInfo(srcFS, src_abs);
if (!srcinfo) {
ERR(errno, "hdfsGetPathInfo failed");
goto out;
}
if (srcinfo->mKind == kObjectKindDirectory) {
ERR(ENOTSUP, "hdfsCopy can't do directories right now");
goto out;
}
a = hdfsOpenFile(srcFS, src_abs, O_RDONLY, 0, 0, 0);
if (!a) {
ERR(errno, "hdfsOpenFile failed");
goto out;
}
b = hdfsOpenFile(dstFS, dst_abs, O_WRONLY, 0, DEFAULT_REPLICATION,
DEFAULT_BLOCK_SIZE);
if (!b) {
ERR(errno, "hdfsOpenFile failed");
goto out;
}
block = malloc(DEFAULT_BLOCK_SIZE);
assert(block);
while (twritten < srcinfo->mSize) {
tSize toread, read, written;
toread = _imin(DEFAULT_BLOCK_SIZE, srcinfo->mSize - twritten);
read = hdfsRead(srcFS, a, block, toread);
if (read == -1) {
ERR(errno, "hdfsRead failed");
goto out;
}
written = hdfsWrite(dstFS, b, block, read);
if (written == -1) {
ERR(errno, "hdfsWrite failed");
goto out;
}
assert(written == read);
twritten += written;
}
res = 0;
out:
if (a)
hdfsCloseFile(srcFS, a);
if (b)
hdfsCloseFile(dstFS, b);
if (src_abs != src)
free(src_abs);
if (dst_abs != dst)
free(dst_abs);
if (block)
free(block);
if (srcinfo)
hdfsFreeFileInfo(srcinfo, 1);
return res;
}
void dir_cpi_impl::sync_copy (saga::impl::void_t & ret,
saga::url src,
saga::url dst,
int flags)
{
instance_data idata (this);
saga::url url(idata->location_);
// handle the files
boost::filesystem::path src_location (idata->location_.get_path(), boost::filesystem::native);
boost::filesystem::path dst_location (src_location);
// complete paths
boost::filesystem::path src_path (src.get_path(), boost::filesystem::native);
boost::filesystem::path dest_path (dst.get_path(), boost::filesystem::native);
if ( ! src_path.has_root_path () )
src_location /= src_path;
else
src_location = src_path;
if ( ! dest_path.has_root_path () )
dst_location /= dest_path;
else
dst_location = dest_path;
bool is_src_dir = false;
if(hdfsExists(fs_, src_location.string().c_str()) == 0)
{
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, src_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_src_dir = true;
hdfsFreeFileInfo(info, 1);
}
// src location refers to a is a directory
if (is_src_dir) {
SAGA_ADAPTOR_THROW("Cannot copy directory at moment.", saga::NotImplemented);
}
else {
//Check to see if dst_location is a directory
bool is_dst_dir = false;
bool dst_exists = false;
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
dst_exists = true;
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, dst_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_dst_dir = true;
hdfsFreeFileInfo(info, 1);
}
else
{
SAGA_ADAPTOR_THROW("Path does not exists!", saga::NoSuccess);
}
if (is_dst_dir)
dst_location /= src_location.leaf();
// remove the file/directory if it exists and we should overwrite
if ((flags & saga::name_space::Overwrite) && dst_exists) {
/* if (is_dst_dir)
fs::remove_all(dst_location);*/
saga_hdfs_delete(fs_, dst_location.string().c_str());
}
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
SAGA_OSSTREAM strm;
strm << "namespace_dir_cpi_impl<Base>::sync_copy: "
"target file already exists: " << dst.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::AlreadyExists);
}
if(hdfsCopy(fs_, src_location.string().c_str(), fs_, dst_location.string().c_str()) != 0)
{
SAGA_OSSTREAM strm;
strm << "namespace_dir_cpi_impl<Base>::sync_copy: "
"target file did not copy successfully: " << dst.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::NoSuccess);
}
}
}
/* constructor */
static PyObject *
pyhdfsFS_open(char* filepath,char mode)
{
pyhdfsFS *object = PyObject_NEW(pyhdfsFS, &pyhdfsFSType);
if (object != NULL)
{
// Parse HDFS information
char* hostport = (char *)malloc(strlen(filepath)+1);
char* buf = NULL;
char* path;
char* portStr;
char* host;
int hdfsPort;
int ret = 0;
ret = sscanf(filepath, "hdfs://%s", hostport);
host = strtok_r(hostport, ":", &buf);
portStr = strtok_r(NULL, "/", &buf);
ret = sscanf(portStr, "%d", &hdfsPort);
object->fs = hdfsConnect(host, hdfsPort);
if (!object->fs)
{
PyErr_SetString(exception, "Cannot connect to host");
return NULL;
}
path = (char*) malloc(strlen(buf)+2);
path[0] = '/';
memcpy(path+1,buf,strlen(buf));
path[strlen(buf)+1] = '\0';
if (mode == 'r')
{
if (hdfsExists(object->fs, path) == 0)
{
object->file = hdfsOpenFile(object->fs, path, O_RDONLY, 0, 0, 0);
}
else
{
char* error_message = (char *)calloc(strlen("Cannot open file for read : ") + strlen(path) + 1,
sizeof(char));
strcat(error_message, "Cannot open file for read : ");
strcat(error_message, path);
PyErr_SetString(exception, error_message);
free(error_message);
return NULL;
}
}
else if (mode == 'w')
{
if (hdfsExists(object->fs, path) == 0)
{
char* error_message = (char *)calloc(strlen("Cannot open file for write : ") + strlen(path) + 1,
sizeof(char));
strcat(error_message, "Cannot open file for write : ");
strcat(error_message, path);
PyErr_SetString(exception, error_message);
free(error_message);
return NULL;
}
else
object->file = hdfsOpenFile(object->fs, path, O_WRONLY, 0, 0, 0);
}
else
{
PyErr_SetString(exception, "Invalid mode");
return NULL;
}
if(!object->file)
{
char* error_message = (char *)calloc(strlen("Cannot open file : ") + strlen(path) + 1,
sizeof(char));
strcat(error_message, "Cannot open file : ");
strcat(error_message, path);
PyErr_SetString(exception, error_message);
free(error_message);
return NULL;
}
free(hostport);
free(path);
}
return (PyObject *)object;
}
int main(int argc, char **argv)
{
char buffer[32];
tSize num_written_bytes;
const char* slashTmp = "/tmp";
int nnPort;
char *rwTemplate, *rwTemplate2, *newDirTemplate,
*appendTemplate, *userTemplate, *rwPath = NULL;
const char* fileContents = "Hello, World!";
const char* nnHost = NULL;
if (argc != 2) {
fprintf(stderr, "usage: test_libwebhdfs_ops <username>\n");
exit(1);
}
struct NativeMiniDfsConf conf = {
.doFormat = 1, .webhdfsEnabled = 1, .namenodeHttpPort = 50070,
};
cluster = nmdCreate(&conf);
if (!cluster) {
fprintf(stderr, "Failed to create the NativeMiniDfsCluster.\n");
exit(1);
}
if (nmdWaitClusterUp(cluster)) {
fprintf(stderr, "Error when waiting for cluster to be ready.\n");
exit(1);
}
if (nmdGetNameNodeHttpAddress(cluster, &nnPort, &nnHost)) {
fprintf(stderr, "Error when retrieving namenode host address.\n");
exit(1);
}
hdfsFS fs = hdfsConnectAsUserNewInstance(nnHost, nnPort, argv[1]);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
{
// Write tests
rwTemplate = strdup("/tmp/helloWorldXXXXXX");
if (!rwTemplate) {
fprintf(stderr, "Failed to create rwTemplate!\n");
exit(1);
}
rwPath = mktemp(rwTemplate);
// hdfsOpenFile
hdfsFile writeFile = hdfsOpenFile(fs, rwPath,
O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", rwPath);
exit(1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", rwPath);
// hdfsWrite
num_written_bytes = hdfsWrite(fs, writeFile, (void*)fileContents,
(int) strlen(fileContents) + 1);
if (num_written_bytes != strlen(fileContents) + 1) {
fprintf(stderr, "Failed to write correct number of bytes - "
"expected %d, got %d\n",
(int)(strlen(fileContents) + 1), (int) num_written_bytes);
exit(1);
}
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
// hdfsTell
tOffset currentPos = -1;
if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
fprintf(stderr,
"Failed to get current file position correctly. Got %"
PRId64 "!\n", currentPos);
exit(1);
}
fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
hdfsCloseFile(fs, writeFile);
// Done test write
}
sleep(1);
{
//Read tests
int available = 0, exists = 0;
// hdfsExists
exists = hdfsExists(fs, rwPath);
if (exists) {
fprintf(stderr, "Failed to validate existence of %s\n", rwPath);
exists = hdfsExists(fs, rwPath);
if (exists) {
fprintf(stderr,
"Still failed to validate existence of %s\n", rwPath);
exit(1);
}
}
hdfsFile readFile = hdfsOpenFile(fs, rwPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", rwPath);
exit(1);
}
if (!hdfsFileIsOpenForRead(readFile)) {
fprintf(stderr, "hdfsFileIsOpenForRead: we just opened a file "
"with O_RDONLY, and it did not show up as 'open for "
"read'\n");
exit(1);
}
available = hdfsAvailable(fs, readFile);
fprintf(stderr, "hdfsAvailable: %d\n", available);
// hdfsSeek, hdfsTell
tOffset seekPos = 1;
if(hdfsSeek(fs, readFile, seekPos)) {
fprintf(stderr, "Failed to seek %s for reading!\n", rwPath);
exit(1);
}
tOffset currentPos = -1;
if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
fprintf(stderr,
"Failed to get current file position correctly! Got %"
PRId64 "!\n", currentPos);
exit(1);
}
fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
if(hdfsSeek(fs, readFile, 0)) {
fprintf(stderr, "Failed to seek %s for reading!\n", rwPath);
exit(1);
}
// hdfsRead
memset(buffer, 0, sizeof(buffer));
tSize num_read_bytes = hdfsRead(fs, readFile, buffer, sizeof(buffer));
if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) {
fprintf(stderr, "Failed to read (direct). "
"Expected %s but got %s (%d bytes)\n",
fileContents, buffer, num_read_bytes);
exit(1);
}
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
if (hdfsSeek(fs, readFile, 0L)) {
fprintf(stderr, "Failed to seek to file start!\n");
exit(1);
}
// hdfsPread
memset(buffer, 0, strlen(fileContents + 1));
num_read_bytes = hdfsPread(fs, readFile, 0, buffer, sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
hdfsCloseFile(fs, readFile);
// Done test read
}
int totalResult = 0;
int result = 0;
{
//Generic file-system operations
char *srcPath = rwPath;
char buffer[256];
const char *resp;
rwTemplate2 = strdup("/tmp/helloWorld2XXXXXX");
if (!rwTemplate2) {
fprintf(stderr, "Failed to create rwTemplate2!\n");
exit(1);
}
char *dstPath = mktemp(rwTemplate2);
newDirTemplate = strdup("/tmp/newdirXXXXXX");
if (!newDirTemplate) {
fprintf(stderr, "Failed to create newDirTemplate!\n");
exit(1);
}
char *newDirectory = mktemp(newDirTemplate);
// hdfsRename
fprintf(stderr, "hdfsRename: %s\n",
((result = hdfsRename(fs, rwPath, dstPath)) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsRename back: %s\n",
((result = hdfsRename(fs, dstPath, srcPath)) ?
"Failed!" : "Success!"));
totalResult += result;
// hdfsCreateDirectory
fprintf(stderr, "hdfsCreateDirectory: %s\n",
((result = hdfsCreateDirectory(fs, newDirectory)) ?
"Failed!" : "Success!"));
totalResult += result;
// hdfsSetReplication
fprintf(stderr, "hdfsSetReplication: %s\n",
((result = hdfsSetReplication(fs, srcPath, 1)) ?
"Failed!" : "Success!"));
totalResult += result;
// hdfsGetWorkingDirectory, hdfsSetWorkingDirectory
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n",
((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ?
buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
const char* path[] = {"/foo", "/foo/bar", "foobar", "//foo/bar//foobar",
"foo//bar", "foo/bar///", "/", "////"};
int i;
for (i = 0; i < 8; i++) {
fprintf(stderr, "hdfsSetWorkingDirectory: %s, %s\n",
((result = hdfsSetWorkingDirectory(fs, path[i])) ?
"Failed!" : "Success!"),
hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer)));
totalResult += result;
}
fprintf(stderr, "hdfsSetWorkingDirectory: %s\n",
((result = hdfsSetWorkingDirectory(fs, slashTmp)) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n",
((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ?
buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
// hdfsGetPathInfo
hdfsFileInfo *fileInfo = NULL;
if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
fprintf(stderr, "Name: %s, ", fileInfo->mName);
fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
fprintf(stderr, "BlockSize: %"PRId64", ", fileInfo->mBlockSize);
fprintf(stderr, "Size: %"PRId64", ", fileInfo->mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
char permissions[10];
permission_disp(fileInfo->mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n",
fileInfo->mPermissions, permissions);
hdfsFreeFileInfo(fileInfo, 1);
} else {
totalResult++;
fprintf(stderr, "hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
}
// hdfsListDirectory
hdfsFileInfo *fileList = 0;
int numEntries = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
int i = 0;
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
fprintf(stderr, "BlockSize: %"PRId64", ", fileList[i].mBlockSize);
fprintf(stderr, "Size: %"PRId64", ", fileList[i].mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
char permissions[10];
permission_disp(fileList[i].mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n",
fileList[i].mPermissions, permissions);
}
hdfsFreeFileInfo(fileList, numEntries);
} else {
if (errno) {
totalResult++;
fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
} else {
fprintf(stderr, "Empty directory!\n");
}
}
char *newOwner = "root";
// Setting tmp dir to 777 so later when connectAsUser nobody,
// we can write to it
short newPerm = 0666;
// hdfsChown
fprintf(stderr, "hdfsChown: %s\n",
((result = hdfsChown(fs, rwPath, NULL, "users")) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChown: %s\n",
((result = hdfsChown(fs, rwPath, newOwner, NULL)) ?
"Failed!" : "Success!"));
totalResult += result;
// hdfsChmod
fprintf(stderr, "hdfsChmod: %s\n",
((result = hdfsChmod(fs, rwPath, newPerm)) ?
"Failed!" : "Success!"));
totalResult += result;
sleep(2);
tTime newMtime = time(NULL);
tTime newAtime = time(NULL);
// utime write
fprintf(stderr, "hdfsUtime: %s\n",
((result = hdfsUtime(fs, rwPath, newMtime, newAtime)) ?
"Failed!" : "Success!"));
totalResult += result;
// chown/chmod/utime read
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, rwPath);
fprintf(stderr, "hdfsChown read: %s\n",
((result = (strcmp(finfo->mOwner, newOwner) != 0)) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChmod read: %s\n",
((result = (finfo->mPermissions != newPerm)) ?
"Failed!" : "Success!"));
totalResult += result;
// will later use /tmp/ as a different user so enable it
fprintf(stderr, "hdfsChmod: %s\n",
((result = hdfsChmod(fs, slashTmp, 0777)) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr,"newMTime=%ld\n",newMtime);
fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
fprintf(stderr, "hdfsUtime read (mtime): %s\n",
((result = (finfo->mLastMod != newMtime / 1000)) ?
"Failed!" : "Success!"));
totalResult += result;
// Clean up
hdfsFreeFileInfo(finfo, 1);
fprintf(stderr, "hdfsDelete: %s\n",
((result = hdfsDelete(fs, newDirectory, 1)) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n",
((result = hdfsDelete(fs, srcPath, 1)) ?
"Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsExists: %s\n",
((result = hdfsExists(fs, newDirectory)) ?
"Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
// Done test generic operations
}
{
// Test Appends
appendTemplate = strdup("/tmp/appendsXXXXXX");
if (!appendTemplate) {
fprintf(stderr, "Failed to create appendTemplate!\n");
exit(1);
}
char *appendPath = mktemp(appendTemplate);
const char* helloBuffer = "Hello,";
hdfsFile writeFile = NULL;
// Create
writeFile = hdfsOpenFile(fs, appendPath, O_WRONLY, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
exit(1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
num_written_bytes = hdfsWrite(fs, writeFile, helloBuffer,
(int) strlen(helloBuffer));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
hdfsCloseFile(fs, writeFile);
fprintf(stderr, "hdfsSetReplication: %s\n",
((result = hdfsSetReplication(fs, appendPath, 1)) ?
"Failed!" : "Success!"));
totalResult += result;
// Re-Open for Append
writeFile = hdfsOpenFile(fs, appendPath, O_WRONLY | O_APPEND, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
exit(1);
}
fprintf(stderr, "Opened %s for appending successfully...\n",
appendPath);
helloBuffer = " World";
num_written_bytes = hdfsWrite(fs, writeFile, helloBuffer,
(int)strlen(helloBuffer) + 1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
hdfsCloseFile(fs, writeFile);
// Check size
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, appendPath);
fprintf(stderr, "fileinfo->mSize: == total %s\n",
((result = (finfo->mSize == strlen("Hello, World") + 1)) ?
"Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
// Read and check data
hdfsFile readFile = hdfsOpenFile(fs, appendPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", appendPath);
exit(1);
}
tSize num_read_bytes = hdfsRead(fs, readFile, buffer, sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
fprintf(stderr, "read == Hello, World %s\n",
(result = (strcmp(buffer, "Hello, World") == 0)) ?
"Success!" : "Failed!");
hdfsCloseFile(fs, readFile);
// Cleanup
fprintf(stderr, "hdfsDelete: %s\n",
((result = hdfsDelete(fs, appendPath, 1)) ?
"Failed!" : "Success!"));
totalResult += result;
// Done test appends
}
totalResult += (hdfsDisconnect(fs) != 0);
{
//
// Now test as connecting as a specific user
// This only meant to test that we connected as that user, not to test
// the actual fs user capabilities. Thus just create a file and read
// the owner is correct.
const char *tuser = "******";
userTemplate = strdup("/tmp/usertestXXXXXX");
if (!userTemplate) {
fprintf(stderr, "Failed to create userTemplate!\n");
exit(1);
}
char* userWritePath = mktemp(userTemplate);
hdfsFile writeFile = NULL;
fs = hdfsConnectAsUserNewInstance("default", 50070, tuser);
if(!fs) {
fprintf(stderr,
"Oops! Failed to connect to hdfs as user %s!\n",tuser);
exit(1);
}
writeFile = hdfsOpenFile(fs, userWritePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", userWritePath);
exit(1);
}
fprintf(stderr, "Opened %s for writing successfully...\n",
userWritePath);
num_written_bytes = hdfsWrite(fs, writeFile, fileContents,
(int)strlen(fileContents) + 1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
hdfsCloseFile(fs, writeFile);
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, userWritePath);
if (finfo) {
fprintf(stderr, "hdfs new file user is correct: %s\n",
((result = (strcmp(finfo->mOwner, tuser) != 0)) ?
"Failed!" : "Success!"));
} else {
fprintf(stderr,
"hdfsFileInfo returned by hdfsGetPathInfo is NULL\n");
result = -1;
}
totalResult += result;
// Cleanup
fprintf(stderr, "hdfsDelete: %s\n",
((result = hdfsDelete(fs, userWritePath, 1)) ?
"Failed!" : "Success!"));
totalResult += result;
// Done test specific user
}
totalResult += (hdfsDisconnect(fs) != 0);
// Shutdown the native minidfscluster
nmdShutdown(cluster);
nmdFree(cluster);
fprintf(stderr, "totalResult == %d\n", totalResult);
if (totalResult != 0) {
return -1;
} else {
return 0;
}
}
void dir_cpi_impl::sync_move (saga::impl::void_t & ret,
saga::url src,
saga::url dest,
int flags)
{
instance_data idata (this);
// verify current working directory is local
saga::url url(idata->location_);
// handle the files
boost::filesystem::path src_location (idata->location_.get_path(), boost::filesystem::native);
boost::filesystem::path dst_location (src_location);
// complete paths
boost::filesystem::path src_path (src.get_path(), boost::filesystem::native);
boost::filesystem::path dest_path (dest.get_path(), boost::filesystem::native);
if ( ! src_path.has_root_path () )
src_location /= src_path;
else
src_location = src_path;
if ( ! dest_path.has_root_path () )
dst_location /= dest_path;
else
dst_location = dest_path;
bool is_src_dir;
bool is_dst_dir;
if(hdfsExists(fs_, src_location.string().c_str()) == 0)
{
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, src_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_src_dir = true;
else
is_src_dir = false;
hdfsFreeFileInfo(info, 1);
}
bool dst_exists = false;
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
dst_exists = true;
//Check to see if it is a directory
hdfsFileInfo *info;
info = hdfsGetPathInfo(fs_, dst_location.string().c_str());
if(info == NULL)
{
SAGA_ADAPTOR_THROW("file_cpi_impl::init failed",
saga::NoSuccess);
}
if(info->mKind == kObjectKindDirectory)
is_dst_dir = true;
else
is_dst_dir = false;
hdfsFreeFileInfo(info, 1);
}
if (!is_src_dir && is_dst_dir)
dst_location /= src_location.leaf();
if ((flags & saga::name_space::Overwrite) && dst_exists) {
/* if (is_dst_dir)
fs::remove_all(dst_location);*/
saga_hdfs_delete(fs_, dst_location.string().c_str());
}
// if destination still exists raise an error
dst_exists = false;
if(hdfsExists(fs_, dst_location.string().c_str()) == 0)
{
SAGA_OSSTREAM strm;
strm << "namespace_dir_cpi_impl<Base>::sync_move: "
"target file already exists: " << dest.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::AlreadyExists);
}
if(hdfsMove(fs_, src_location.string().c_str(), fs_, dst_location.string().c_str()) != 0)
{
SAGA_ADAPTOR_THROW("Unable to move files", saga::NoSuccess);
}
}
bool Hdfs::exists(string path)
{
// there doesn't appear to be an error code for this function. Simply 0 if it exists and -1 if
// it doesn't
return hdfsExists(_getFs(), path.data()) == 0;
}
static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs)
{
char prefix[256], tmp[256];
hdfsFile file;
int ret, expected;
hdfsFileInfo *fileInfo;
snprintf(prefix, sizeof(prefix), "/tlhData%04d", ti->threadIdx);
if (hdfsExists(fs, prefix) == 0) {
EXPECT_ZERO(hdfsDelete(fs, prefix, 1));
}
EXPECT_ZERO(hdfsCreateDirectory(fs, prefix));
snprintf(tmp, sizeof(tmp), "%s/file", prefix);
EXPECT_NONNULL(hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0));
file = hdfsOpenFile(fs, tmp, O_WRONLY, 0, 0, 0);
EXPECT_NONNULL(file);
/* TODO: implement writeFully and use it here */
expected = (int)strlen(prefix);
ret = hdfsWrite(fs, file, prefix, expected);
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
"it wrote %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(hdfsFlush(fs, file));
EXPECT_ZERO(hdfsCloseFile(fs, file));
/* Let's re-open the file for reading */
file = hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0);
EXPECT_NONNULL(file);
/* TODO: implement readFully and use it here */
ret = hdfsRead(fs, file, tmp, sizeof(tmp));
if (ret < 0) {
ret = errno;
fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
return ret;
}
if (ret != expected) {
fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
"it read %d\n", ret, expected);
return EIO;
}
EXPECT_ZERO(memcmp(prefix, tmp, expected));
EXPECT_ZERO(hdfsCloseFile(fs, file));
snprintf(tmp, sizeof(tmp), "%s/file", prefix);
EXPECT_NONZERO(hdfsChown(fs, tmp, NULL, NULL));
EXPECT_ZERO(hdfsChown(fs, tmp, NULL, "doop"));
fileInfo = hdfsGetPathInfo(fs, tmp);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("doop", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, tmp, "ha", "doop2"));
fileInfo = hdfsGetPathInfo(fs, tmp);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsChown(fs, tmp, "ha2", NULL));
fileInfo = hdfsGetPathInfo(fs, tmp);
EXPECT_NONNULL(fileInfo);
EXPECT_ZERO(strcmp("ha2", fileInfo->mOwner));
EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup));
hdfsFreeFileInfo(fileInfo, 1);
EXPECT_ZERO(hdfsDelete(fs, prefix, 1));
return 0;
}
int main(int argc, char **argv) {
const char *writePath = "/tmp/testfile.txt";
const char *fileContents = "Hello, World!";
const char *readPath = "/tmp/testfile.txt";
const char *srcPath = "/tmp/testfile.txt";
const char *dstPath = "/tmp/testfile2.txt";
const char *slashTmp = "/tmp";
const char *newDirectory = "/tmp/newdir";
const char *newOwner = "root";
const char *tuser = "******";
const char *appendPath = "/tmp/appends";
const char *userPath = "/tmp/usertestfile.txt";
char buffer[32], buffer2[256], rdbuffer[32];
tSize num_written_bytes, num_read_bytes;
hdfsFS fs, lfs;
hdfsFile writeFile, readFile, localFile, appendFile, userFile;
tOffset currentPos, seekPos;
int exists, totalResult, result, numEntries, i, j;
const char *resp;
hdfsFileInfo *fileInfo, *fileList, *finfo;
char *buffer3;
char permissions[10];
char ***hosts;
short newPerm = 0666;
tTime newMtime, newAtime;
fs = hdfsConnectNewInstance("default", 0);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
lfs = hdfsConnectNewInstance(NULL, 0);
if(!lfs) {
fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
exit(-1);
}
{
//Write tests
writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
num_written_bytes =
hdfsWrite(fs, writeFile, (void*)fileContents,
(tSize)(strlen(fileContents)+1));
if (num_written_bytes != strlen(fileContents) + 1) {
fprintf(stderr, "Failed to write correct number of bytes - expected %d, got %d\n",
(int)(strlen(fileContents) + 1), (int)num_written_bytes);
exit(-1);
}
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
currentPos = -1;
if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
fprintf(stderr,
"Failed to get current file position correctly! Got %" PRId64 "!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
if (hdfsHFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'hflush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "HFlushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
}
{
//Read tests
exists = hdfsExists(fs, readPath);
if (exists) {
fprintf(stderr, "Failed to validate existence of %s\n", readPath);
exit(-1);
}
readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", readPath);
exit(-1);
}
if (!hdfsFileIsOpenForRead(readFile)) {
fprintf(stderr, "hdfsFileIsOpenForRead: we just opened a file "
"with O_RDONLY, and it did not show up as 'open for "
"read'\n");
exit(-1);
}
fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));
seekPos = 1;
if(hdfsSeek(fs, readFile, seekPos)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
currentPos = -1;
if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
fprintf(stderr,
"Failed to get current file position correctly! Got %" PRId64 "!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
if (!hdfsFileUsesDirectRead(readFile)) {
fprintf(stderr, "Direct read support incorrectly not detected "
"for HDFS filesystem\n");
exit(-1);
}
fprintf(stderr, "Direct read support detected for HDFS\n");
// Test the direct read path
if(hdfsSeek(fs, readFile, 0)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
memset(buffer, 0, sizeof(buffer));
num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) {
fprintf(stderr, "Failed to read (direct). Expected %s but got %s (%d bytes)\n",
fileContents, buffer, num_read_bytes);
exit(-1);
}
fprintf(stderr, "Read (direct) following %d bytes:\n%s\n",
num_read_bytes, buffer);
if (hdfsSeek(fs, readFile, 0L)) {
fprintf(stderr, "Failed to seek to file start!\n");
exit(-1);
}
// Disable the direct read path so that we really go through the slow
// read path
hdfsFileDisableDirectRead(readFile);
num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
memset(buffer, 0, strlen(fileContents + 1));
num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
hdfsCloseFile(fs, readFile);
// Test correct behaviour for unsupported filesystems
localFile = hdfsOpenFile(lfs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!localFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
num_written_bytes = hdfsWrite(lfs, localFile, (void*)fileContents,
(tSize)(strlen(fileContents) + 1));
hdfsCloseFile(lfs, localFile);
localFile = hdfsOpenFile(lfs, writePath, O_RDONLY, 0, 0, 0);
if (hdfsFileUsesDirectRead(localFile)) {
fprintf(stderr, "Direct read support incorrectly detected for local "
"filesystem\n");
exit(-1);
}
hdfsCloseFile(lfs, localFile);
}
totalResult = 0;
result = 0;
{
//Generic file-system operations
fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsGetDefaultBlockSize: %" PRId64 "\n", hdfsGetDefaultBlockSize(fs));
fprintf(stderr, "hdfsGetCapacity: %" PRId64 "\n", hdfsGetCapacity(fs));
fprintf(stderr, "hdfsGetUsed: %" PRId64 "\n", hdfsGetUsed(fs));
fileInfo = NULL;
if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
fprintf(stderr, "Name: %s, ", fileInfo->mName);
fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
fprintf(stderr, "BlockSize: %" PRId64 ", ", fileInfo->mBlockSize);
fprintf(stderr, "Size: %" PRId64 ", ", fileInfo->mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
permission_disp(fileInfo->mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
hdfsFreeFileInfo(fileInfo, 1);
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
}
fileList = 0;
fileList = hdfsListDirectory(fs, newDirectory, &numEntries);
if (!(fileList == NULL && numEntries == 0 && !errno)) {
fprintf(stderr, "waah! hdfsListDirectory for empty %s - FAILED!\n", newDirectory);
totalResult++;
} else {
fprintf(stderr, "hdfsListDirectory for empty %s - SUCCESS!\n", newDirectory);
}
fileList = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
fprintf(stderr, "BlockSize: %" PRId64 ", ", fileList[i].mBlockSize);
fprintf(stderr, "Size: %" PRId64 ", ", fileList[i].mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
permission_disp(fileList[i].mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
}
hdfsFreeFileInfo(fileList, numEntries);
} else {
if (errno) {
totalResult++;
fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
} else {
fprintf(stderr, "Empty directory!\n");
}
}
hosts = hdfsGetHosts(fs, srcPath, 0, 1);
if(hosts) {
fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
i=0;
while(hosts[i]) {
j = 0;
while(hosts[i][j]) {
fprintf(stderr,
"\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
++j;
}
++i;
}
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
}
// setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
// chown write
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// chmod write
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
sleep(2);
newMtime = time(NULL);
newAtime = time(NULL);
// utime write
fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// chown/chmod/utime read
finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner))) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// will later use /tmp/ as a different user so enable it
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr,"newMTime=%ld\n",newMtime);
fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// No easy way to turn on access times from hdfs_test right now
// fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) != 0 ? "Failed!" : "Success!"));
// totalResult += result;
hdfsFreeFileInfo(finfo, 1);
// Clean up
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) != 0 ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
}
{
// TEST APPENDS
// CREATE
appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY, 0, 0, 0);
if(!appendFile) {
fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
buffer3 = "Hello,";
num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3,
(tSize)strlen(buffer3));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, appendFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", appendPath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", appendPath);
hdfsCloseFile(fs, appendFile);
// RE-OPEN
appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY|O_APPEND, 0, 0, 0);
if(!appendFile) {
fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
buffer3 = " World";
num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3,
(tSize)(strlen(buffer3) + 1));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, appendFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", appendPath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", appendPath);
hdfsCloseFile(fs, appendFile);
// CHECK size
finfo = hdfsGetPathInfo(fs, appendPath);
fprintf(stderr, "fileinfo->mSize: == total %s\n", ((result = (finfo->mSize == (tOffset)(strlen("Hello, World") + 1))) == 1 ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
// READ and check data
readFile = hdfsOpenFile(fs, appendPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", appendPath);
exit(-1);
}
num_read_bytes = hdfsRead(fs, readFile, (void*)rdbuffer, sizeof(rdbuffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, rdbuffer);
fprintf(stderr, "read == Hello, World %s\n", ((result = (strcmp(rdbuffer, "Hello, World"))) == 0 ? "Success!" : "Failed!"));
hdfsCloseFile(fs, readFile);
// DONE test appends
}
totalResult += (hdfsDisconnect(fs) != 0);
{
//
// Now test as connecting as a specific user
// This is only meant to test that we connected as that user, not to test
// the actual fs user capabilities. Thus just create a file and read
// the owner is correct.
fs = hdfsConnectAsUserNewInstance("default", 0, tuser);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
exit(-1);
}
userFile = hdfsOpenFile(fs, userPath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!userFile) {
fprintf(stderr, "Failed to open %s for writing!\n", userPath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", userPath);
num_written_bytes = hdfsWrite(fs, userFile, (void*)fileContents,
(tSize)(strlen(fileContents)+1));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, userFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", userPath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", userPath);
hdfsCloseFile(fs, userFile);
finfo = hdfsGetPathInfo(fs, userPath);
fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser))) != 0 ? "Failed!" : "Success!"));
totalResult += result;
}
totalResult += (hdfsDisconnect(fs) != 0);
if (totalResult != 0) {
return -1;
} else {
return 0;
}
}
int main(int argc, char **argv) {
hdfsFS fs = hdfsConnect("default", 0);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
hdfsFS lfs = hdfsConnect(NULL, 0);
if(!lfs) {
fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
exit(-1);
}
const char* writePath = "/tmp/testfile.txt";
{
//Write tests
hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
char* buffer = "Hello, World!";
tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
tOffset currentPos = -1;
if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
fprintf(stderr,
"Failed to get current file position correctly! Got %ld!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %ld\n", currentPos);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
}
{
//Read tests
const char* readPath = "/tmp/testfile.txt";
int exists = hdfsExists(fs, readPath);
if (exists) {
fprintf(stderr, "Failed to validate existence of %s\n", readPath);
exit(-1);
}
hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", readPath);
exit(-1);
}
fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));
tOffset seekPos = 1;
if(hdfsSeek(fs, readFile, seekPos)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
tOffset currentPos = -1;
if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
fprintf(stderr,
"Failed to get current file position correctly! Got %ld!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %ld\n", currentPos);
static char buffer[32];
tSize num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
hdfsCloseFile(fs, readFile);
}
int totalResult = 0;
int result = 0;
{
//Generic file-system operations
const char* srcPath = "/tmp/testfile.txt";
const char* dstPath = "/tmp/testfile2.txt";
fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
const char* slashTmp = "/tmp";
const char* newDirectory = "/tmp/newdir";
fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) ? "Failed!" : "Success!"));
totalResult += result;
char buffer[256];
const char *resp;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsGetDefaultBlockSize: %ld\n", hdfsGetDefaultBlockSize(fs));
fprintf(stderr, "hdfsGetCapacity: %ld\n", hdfsGetCapacity(fs));
fprintf(stderr, "hdfsGetUsed: %ld\n", hdfsGetUsed(fs));
hdfsFileInfo *fileInfo = NULL;
if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
fprintf(stderr, "Name: %s, ", fileInfo->mName);
fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
fprintf(stderr, "BlockSize: %ld, ", fileInfo->mBlockSize);
fprintf(stderr, "Size: %ld, ", fileInfo->mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
char permissions[10];
permission_disp(fileInfo->mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
hdfsFreeFileInfo(fileInfo, 1);
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
}
hdfsFileInfo *fileList = 0;
int numEntries = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
int i = 0;
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
fprintf(stderr, "BlockSize: %ld, ", fileList[i].mBlockSize);
fprintf(stderr, "Size: %ld, ", fileList[i].mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
char permissions[10];
permission_disp(fileList[i].mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
}
hdfsFreeFileInfo(fileList, numEntries);
} else {
if (errno) {
totalResult++;
fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
} else {
fprintf(stderr, "Empty directory!\n");
}
}
char*** hosts = hdfsGetHosts(fs, srcPath, 0, 1);
if(hosts) {
fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
int i=0;
while(hosts[i]) {
int j = 0;
while(hosts[i][j]) {
fprintf(stderr,
"\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
++j;
}
++i;
}
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
}
char *newOwner = "root";
// setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
short newPerm = 0666;
// chown write
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) ? "Failed!" : "Success!"));
totalResult += result;
// chmod write
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) ? "Failed!" : "Success!"));
totalResult += result;
sleep(2);
tTime newMtime = time(NULL);
tTime newAtime = time(NULL);
// utime write
fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) ? "Failed!" : "Success!"));
totalResult += result;
// chown/chmod/utime read
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!"));
totalResult += result;
// will later use /tmp/ as a different user so enable it
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr,"newMTime=%ld\n",newMtime);
fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) ? "Failed!" : "Success!"));
totalResult += result;
// No easy way to turn on access times from hdfs_test right now
// fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) ? "Failed!" : "Success!"));
// totalResult += result;
hdfsFreeFileInfo(finfo, 1);
// Clean up
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
}
totalResult += (hdfsDisconnect(fs) != 0);
{
//
// Now test as connecting as a specific user
// This is only meant to test that we connected as that user, not to test
// the actual fs user capabilities. Thus just create a file and read
// the owner is correct.
const char *tuser = "******";
const char* writePath = "/tmp/usertestfile.txt";
const char **groups = (const char**)malloc(sizeof(char*)* 2);
groups[0] = "users";
groups[1] = "nobody";
fs = hdfsConnectAsUser("default", 0, tuser, groups, 2);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
exit(-1);
}
hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
char* buffer = "Hello, World!";
tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!"));
totalResult += result;
}
totalResult += (hdfsDisconnect(fs) != 0);
if (totalResult != 0) {
return -1;
} else {
return 0;
}
}
int move_to_trash(const char *item, hdfsFS userFS) {
// retrieve dfs specific data
dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
// check params and the context var
assert(item);
assert(dfs);
assert('/' == *item);
assert(rindex(item,'/') >= 0);
char fname[4096]; // or last element of the directory path
char parent_dir[4096]; // the directory the fname resides in
if (strlen(item) > sizeof(fname) - strlen(TrashDir)) {
ERROR("Buffer too small to accomodate path of len %d", (int)strlen(item));
return -EIO;
}
// separate the file name and the parent directory of the item to be deleted
{
int length_of_parent_dir = rindex(item, '/') - item ;
int length_of_fname = strlen(item) - length_of_parent_dir - 1; // the '/'
// note - the below strncpys should be safe from overflow because of the check on item's string length above.
strncpy(parent_dir, item, length_of_parent_dir);
parent_dir[length_of_parent_dir ] = 0;
strncpy(fname, item + length_of_parent_dir + 1, strlen(item));
fname[length_of_fname + 1] = 0;
}
// create the target trash directory
char trash_dir[4096];
if (snprintf(trash_dir, sizeof(trash_dir), "%s%s", TrashDir, parent_dir)
>= sizeof trash_dir) {
ERROR("Move to trash error target not big enough for %s", item);
return -EIO;
}
// create the target trash directory in trash (if needed)
if ( hdfsExists(userFS, trash_dir)) {
// make the directory to put it in in the Trash - NOTE
// hdfsCreateDirectory also creates parents, so Current will be created if it does not exist.
if (hdfsCreateDirectory(userFS, trash_dir)) {
return -EIO;
}
}
//
// if the target path in Trash already exists, then append with
// a number. Start from 1.
//
char target[4096];
int j ;
if ( snprintf(target, sizeof target,"%s/%s",trash_dir, fname) >= sizeof target) {
ERROR("Move to trash error target not big enough for %s", item);
return -EIO;
}
// NOTE: this loop differs from the java version by capping the #of tries
for (j = 1; ! hdfsExists(userFS, target) && j < TRASH_RENAME_TRIES ; j++) {
if (snprintf(target, sizeof target,"%s/%s.%d",trash_dir, fname, j) >= sizeof target) {
ERROR("Move to trash error target not big enough for %s", item);
return -EIO;
}
}
if (hdfsRename(userFS, item, target)) {
ERROR("Trying to rename %s to %s", item, target);
return -EIO;
}
return 0;
}
