qioerr hdfs_get_owners_for_bytes(qio_file_t* file, hdfs_block_byte_map_t** locs, int* out_num_blocks, char** locale_array, int num_locales, off_t start_byte, off_t len)
{
int i;
int j = 0;
int k;
qioerr err = 0;
char* tmp;
int rnd;
int block_count = 0;
hdfs_block_byte_map_t* loc = NULL;
char*** info = NULL;
hdfsFileInfo* f_info = hdfsGetPathInfo(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm);
if (start_byte == 0 && len == -1) // We want the whole thing
info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, f_info->mSize);
else info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, start_byte + len);
while(info[block_count] != NULL) { // Get the number of blocks that we have
block_count++;
}
loc = (hdfs_block_byte_map_t*)qio_calloc(sizeof(hdfs_block_byte_map_t), block_count);
CREATE_ERROR((!info), err, EREMOTEIO, "Unable to get host for HDFS", end);
for (i = 0; info[i] != NULL; i++) { // Assign block owners
rnd = rand() % f_info->mReplication; // pick an owner
if (info[i][rnd]) {// Valid access
tmp = get_locale_name(info[i][rnd]); // strip off .___
for (k = 0; k < num_locales; k++) { // Now find the owner
if (strcmp(tmp, locale_array[k]) == 0) {
loc[i].locale_id = k; // return locale ID for that name
break;
}
}
loc[i].start_byte = (off_t)(i*f_info->mBlockSize);
loc[i].len = (off_t)(f_info->mBlockSize);
j++;
} else {
QIO_GET_CONSTANT_ERROR(err, EINVAL, "Unable to find address for blocks in hdfs_get_owners_for_bytes");
qio_free(loc);
*locs = NULL;
*out_num_blocks = 0;
goto end;
}
}
*locs = loc;
*out_num_blocks = j;
end:
return err;
}
QStringList Hdfs::getLocations(string path, long start, long length, const bool sort)
{
char*** h = hdfsGetHosts(_getFs(), path.data(), start, length);
if (h == NULL)
{
throw ios_base::failure("Error getting block locations from HDFS.");
}
QSet<QString> hosts;
int i = 0;
while (h[i] != NULL)
{
int j = 0;
while (h[i][j] != NULL)
{
char* host = h[i][j];
hosts.insert(host);
j++;
}
i++;
}
QStringList hostsList(hosts.toList());
if (sort)
{
hostsList.sort();
}
return hostsList;
}
qioerr hdfs_locales_for_range(void* file, off_t start_byte, off_t end_byte, const char*** loc_names_out, int* num_locs_out, void* fs)
{
int i = 0;
int j = 0;
char*** info = NULL;
info = hdfsGetHosts(to_hdfs_fs(fs)->hfs, to_hdfs_file(file)->pathnm, start_byte, end_byte);
// unable to get hosts for this byte range
if (!info || !info[0]) {
*num_locs_out = 0;
hdfsFreeHosts(info);
QIO_RETURN_CONSTANT_ERROR(EREMOTEIO, "Unable to get owners for byterange");
}
while(info[0][i]) {
info[0][i] = get_locale_name(info[0][i]);
i++;
}
*num_locs_out = i - 1;
*loc_names_out = (const char**)info[0];
// Free the other hosts that we don't need
for (i = 1; info[i]; i++) {
for (j = 0; info[i][j]; j++)
qio_free(info[i][j]);
qio_free(info[i]);
}
return 0;
}
void libhdfsconnector::ouputhosts(const char * rfile)
{
if (!fs)
{
fprintf(stderr, "Could not connect to hdfs");
return;
}
char*** hosts = hdfsGetHosts(fs, rfile, 1, 1);
if (hosts)
{
fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
int i = 0;
while (hosts[i])
{
int j = 0;
while (hosts[i][j])
{
fprintf(stderr, "\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
++j;
}
++i;
}
}
}
/**
* call-seq:
* hdfs.get_hosts(path, start, length) -> retval
*
* Returns the hostnames of the DataNodes which serve the portion of the file
* between the provided start and length bytes. Raises a DFSException if this
* fails.
*/
VALUE HDFS_File_System_get_hosts(VALUE self, VALUE path, VALUE start,
VALUE length) {
FSData* data = get_FSData(self);
char*** hosts = hdfsGetHosts(data->fs, StringValuePtr(path), NUM2LONG(start),
NUM2LONG(length));
if (hosts == NULL) {
rb_raise(e_dfs_exception,
"Error while retrieving hosts at path: %s, start: %s, length: %lu: %s",
StringValuePtr(path), StringValuePtr(start), NUM2LONG(length),
get_error(errno));
return Qnil;
}
// Builds a Ruby Array object out of the hosts reported by HDFS.
VALUE hosts_array = rb_ary_new();
size_t i, j;
for (i = 0; hosts[i]; i++) {
VALUE cur_block_hosts = rb_ary_new();
for (j = 0; hosts[i][j]; j++) {
rb_ary_push(cur_block_hosts, rb_str_new2(hosts[i][j]));
}
rb_ary_push(hosts_array, cur_block_hosts);
}
return hosts_array;
}
int main(int argc, char **argv) {
hdfsFS fs = hdfsConnect("default", 0);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
hdfsFS lfs = hdfsConnect(NULL, 0);
if(!lfs) {
fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
exit(-1);
}
const char* writePath = "/tmp/testfile.txt";
{
//Write tests
hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
char* buffer = "Hello, World!";
tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
tOffset currentPos = -1;
if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
fprintf(stderr,
"Failed to get current file position correctly! Got %ld!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %ld\n", currentPos);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
}
{
//Read tests
const char* readPath = "/tmp/testfile.txt";
int exists = hdfsExists(fs, readPath);
if (exists) {
fprintf(stderr, "Failed to validate existence of %s\n", readPath);
exit(-1);
}
hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", readPath);
exit(-1);
}
fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));
tOffset seekPos = 1;
if(hdfsSeek(fs, readFile, seekPos)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
tOffset currentPos = -1;
if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
fprintf(stderr,
"Failed to get current file position correctly! Got %ld!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %ld\n", currentPos);
static char buffer[32];
tSize num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
hdfsCloseFile(fs, readFile);
}
int totalResult = 0;
int result = 0;
{
//Generic file-system operations
const char* srcPath = "/tmp/testfile.txt";
const char* dstPath = "/tmp/testfile2.txt";
fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
const char* slashTmp = "/tmp";
const char* newDirectory = "/tmp/newdir";
fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) ? "Failed!" : "Success!"));
totalResult += result;
char buffer[256];
const char *resp;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsGetDefaultBlockSize: %ld\n", hdfsGetDefaultBlockSize(fs));
fprintf(stderr, "hdfsGetCapacity: %ld\n", hdfsGetCapacity(fs));
fprintf(stderr, "hdfsGetUsed: %ld\n", hdfsGetUsed(fs));
hdfsFileInfo *fileInfo = NULL;
if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
fprintf(stderr, "Name: %s, ", fileInfo->mName);
fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
fprintf(stderr, "BlockSize: %ld, ", fileInfo->mBlockSize);
fprintf(stderr, "Size: %ld, ", fileInfo->mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
char permissions[10];
permission_disp(fileInfo->mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
hdfsFreeFileInfo(fileInfo, 1);
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
}
hdfsFileInfo *fileList = 0;
int numEntries = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
int i = 0;
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
fprintf(stderr, "BlockSize: %ld, ", fileList[i].mBlockSize);
fprintf(stderr, "Size: %ld, ", fileList[i].mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
char permissions[10];
permission_disp(fileList[i].mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
}
hdfsFreeFileInfo(fileList, numEntries);
} else {
if (errno) {
totalResult++;
fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
} else {
fprintf(stderr, "Empty directory!\n");
}
}
char*** hosts = hdfsGetHosts(fs, srcPath, 0, 1);
if(hosts) {
fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
int i=0;
while(hosts[i]) {
int j = 0;
while(hosts[i][j]) {
fprintf(stderr,
"\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
++j;
}
++i;
}
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
}
char *newOwner = "root";
// setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
short newPerm = 0666;
// chown write
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) ? "Failed!" : "Success!"));
totalResult += result;
// chmod write
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) ? "Failed!" : "Success!"));
totalResult += result;
sleep(2);
tTime newMtime = time(NULL);
tTime newAtime = time(NULL);
// utime write
fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) ? "Failed!" : "Success!"));
totalResult += result;
// chown/chmod/utime read
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!"));
totalResult += result;
// will later use /tmp/ as a different user so enable it
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr,"newMTime=%ld\n",newMtime);
fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) ? "Failed!" : "Success!"));
totalResult += result;
// No easy way to turn on access times from hdfs_test right now
// fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) ? "Failed!" : "Success!"));
// totalResult += result;
hdfsFreeFileInfo(finfo, 1);
// Clean up
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath)) ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
}
totalResult += (hdfsDisconnect(fs) != 0);
{
//
// Now test as connecting as a specific user
// This is only meant to test that we connected as that user, not to test
// the actual fs user capabilities. Thus just create a file and read
// the owner is correct.
const char *tuser = "******";
const char* writePath = "/tmp/usertestfile.txt";
const char **groups = (const char**)malloc(sizeof(char*)* 2);
groups[0] = "users";
groups[1] = "nobody";
fs = hdfsConnectAsUser("default", 0, tuser, groups, 2);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
exit(-1);
}
hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
char* buffer = "Hello, World!";
tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!"));
totalResult += result;
}
totalResult += (hdfsDisconnect(fs) != 0);
if (totalResult != 0) {
return -1;
} else {
return 0;
}
}
NABoolean HHDFSFileStats::populate(hdfsFS fs, hdfsFileInfo *fileInfo,
Int32& samples,
NABoolean doEstimation, char recordTerminator,
NABoolean isSequenceFile)
{
NABoolean result = TRUE;
// copy fields from fileInfo
fileName_ = fileInfo->mName;
replication_ = (Int32) fileInfo->mReplication;
totalSize_ = (Int64) fileInfo->mSize;
blockSize_ = (Int64) fileInfo->mBlockSize;
modificationTS_ = fileInfo->mLastMod;
numFiles_ = 1;
isSequenceFile_ = isSequenceFile;
Int64 sampleBufferSize = MINOF(blockSize_, 65536);
NABoolean sortHosts = (CmpCommon::getDefault(HIVE_SORT_HDFS_HOSTS) == DF_ON);
sampleBufferSize = MINOF(sampleBufferSize,totalSize_/10);
if (doEstimation && sampleBufferSize > 100) {
//
// Open the hdfs file to estimate record length. Read one block at
// a time searching for <s> instances of record separators. Stop reading
// when either <s> instances have been found or a partial number of
// instances have and we have exhausted all data content in the block.
// We will keep reading if the current block does not contain
// any instance of the record separator.
//
hdfsFile file =
hdfsOpenFile(fs, fileInfo->mName,
O_RDONLY,
sampleBufferSize, // buffer size
0, // replication, take the default size
fileInfo->mBlockSize // blocksize
);
if ( file != NULL ) {
tOffset offset = 0;
tSize bufLen = sampleBufferSize;
char* buffer = new (heap_) char[bufLen+1];
buffer[bufLen] = 0; // extra null at the end to protect strchr()
// to run over the buffer.
NABoolean sampleDone = FALSE;
Int32 totalSamples = 10;
Int32 totalLen = 0;
while (!sampleDone) {
tSize szRead = hdfsPread(fs, file, offset, buffer, bufLen);
char* pos = NULL;
if ( szRead > 0 ) {
//if (isSequenceFile && offset==0 && memcmp(buffer, "SEQ6", 4) == 0)
// isSequenceFile_ = TRUE;
char* start = buffer;
for (Int32 i=0; i<totalSamples; i++ ) {
if ( (pos=strchr(start, recordTerminator)) ) {
totalLen += pos - start + 1 + offset;
samples++;
start = pos+1;
if ( start > buffer + bufLen ) {
sampleDone = TRUE;
break;
}
} else
break;
}
if ( samples > 0 )
break;
else
offset += bufLen;
} else
break; // fail to read any bytes. Bail out.
}
NADELETEBASIC(buffer, heap_);
if ( samples > 0 ) {
sampledBytes_ += totalLen;
sampledRows_ += samples;
}
hdfsCloseFile(fs, file);
} else {
// can not do hdfs open on the file. Assume the file is empty.
}
}
if (blockSize_)
{
numBlocks_ = totalSize_ / blockSize_;
if (totalSize_ % blockSize_ > 0)
numBlocks_++; // partial block at the end
}
else
{
CMPASSERT(blockSize_);
// TBD:DIAGS
result = FALSE;
}
if ( totalSize_ > 0 )
{
blockHosts_ = new(heap_) HostId[replication_*numBlocks_];
// walk through blocks and record their locations
tOffset o = 0;
Int64 blockNum;
for (blockNum=0; blockNum < numBlocks_ && result; blockNum++)
{
char*** blockHostNames = hdfsGetHosts(fs,
fileInfo->mName,
o,
fileInfo->mBlockSize);
o += blockSize_;
if (blockHostNames == NULL)
{
CMPASSERT(blockHostNames);
// TBD:DIAGS
result = FALSE;
}
else
{
char **h = *blockHostNames;
HostId hostId;
for (Int32 r=0; r<replication_; r++)
{
if (h[r])
hostId = HHDFSMasterHostList::getHostNum(h[r]);
else
hostId = HHDFSMasterHostList::InvalidHostId;
blockHosts_[r*numBlocks_+blockNum] = hostId;
}
if (sortHosts)
sortHostArray(blockHosts_,
(Int32) numBlocks_,
replication_,
getFileName());
}
hdfsFreeHosts(blockHostNames);
}
}
return result;
}
int main(int argc, char **argv) {
const char *writePath = "/tmp/testfile.txt";
const char *fileContents = "Hello, World!";
const char *readPath = "/tmp/testfile.txt";
const char *srcPath = "/tmp/testfile.txt";
const char *dstPath = "/tmp/testfile2.txt";
const char *slashTmp = "/tmp";
const char *newDirectory = "/tmp/newdir";
const char *newOwner = "root";
const char *tuser = "******";
const char *appendPath = "/tmp/appends";
const char *userPath = "/tmp/usertestfile.txt";
char buffer[32], buffer2[256], rdbuffer[32];
tSize num_written_bytes, num_read_bytes;
hdfsFS fs, lfs;
hdfsFile writeFile, readFile, localFile, appendFile, userFile;
tOffset currentPos, seekPos;
int exists, totalResult, result, numEntries, i, j;
const char *resp;
hdfsFileInfo *fileInfo, *fileList, *finfo;
char *buffer3;
char permissions[10];
char ***hosts;
short newPerm = 0666;
tTime newMtime, newAtime;
fs = hdfsConnectNewInstance("default", 0);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
lfs = hdfsConnectNewInstance(NULL, 0);
if(!lfs) {
fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
exit(-1);
}
{
//Write tests
writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
num_written_bytes =
hdfsWrite(fs, writeFile, (void*)fileContents,
(tSize)(strlen(fileContents)+1));
if (num_written_bytes != strlen(fileContents) + 1) {
fprintf(stderr, "Failed to write correct number of bytes - expected %d, got %d\n",
(int)(strlen(fileContents) + 1), (int)num_written_bytes);
exit(-1);
}
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
currentPos = -1;
if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
fprintf(stderr,
"Failed to get current file position correctly! Got %" PRId64 "!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
if (hdfsFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", writePath);
if (hdfsHFlush(fs, writeFile)) {
fprintf(stderr, "Failed to 'hflush' %s\n", writePath);
exit(-1);
}
fprintf(stderr, "HFlushed %s successfully!\n", writePath);
hdfsCloseFile(fs, writeFile);
}
{
//Read tests
exists = hdfsExists(fs, readPath);
if (exists) {
fprintf(stderr, "Failed to validate existence of %s\n", readPath);
exit(-1);
}
readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", readPath);
exit(-1);
}
if (!hdfsFileIsOpenForRead(readFile)) {
fprintf(stderr, "hdfsFileIsOpenForRead: we just opened a file "
"with O_RDONLY, and it did not show up as 'open for "
"read'\n");
exit(-1);
}
fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));
seekPos = 1;
if(hdfsSeek(fs, readFile, seekPos)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
currentPos = -1;
if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
fprintf(stderr,
"Failed to get current file position correctly! Got %" PRId64 "!\n",
currentPos);
exit(-1);
}
fprintf(stderr, "Current position: %" PRId64 "\n", currentPos);
if (!hdfsFileUsesDirectRead(readFile)) {
fprintf(stderr, "Direct read support incorrectly not detected "
"for HDFS filesystem\n");
exit(-1);
}
fprintf(stderr, "Direct read support detected for HDFS\n");
// Test the direct read path
if(hdfsSeek(fs, readFile, 0)) {
fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
exit(-1);
}
memset(buffer, 0, sizeof(buffer));
num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) {
fprintf(stderr, "Failed to read (direct). Expected %s but got %s (%d bytes)\n",
fileContents, buffer, num_read_bytes);
exit(-1);
}
fprintf(stderr, "Read (direct) following %d bytes:\n%s\n",
num_read_bytes, buffer);
if (hdfsSeek(fs, readFile, 0L)) {
fprintf(stderr, "Failed to seek to file start!\n");
exit(-1);
}
// Disable the direct read path so that we really go through the slow
// read path
hdfsFileDisableDirectRead(readFile);
num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
memset(buffer, 0, strlen(fileContents + 1));
num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer,
sizeof(buffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, buffer);
hdfsCloseFile(fs, readFile);
// Test correct behaviour for unsupported filesystems
localFile = hdfsOpenFile(lfs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!localFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writePath);
exit(-1);
}
num_written_bytes = hdfsWrite(lfs, localFile, (void*)fileContents,
(tSize)(strlen(fileContents) + 1));
hdfsCloseFile(lfs, localFile);
localFile = hdfsOpenFile(lfs, writePath, O_RDONLY, 0, 0, 0);
if (hdfsFileUsesDirectRead(localFile)) {
fprintf(stderr, "Direct read support incorrectly detected for local "
"filesystem\n");
exit(-1);
}
hdfsCloseFile(lfs, localFile);
}
totalResult = 0;
result = 0;
{
//Generic file-system operations
fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!"));
totalResult += (resp ? 0 : 1);
fprintf(stderr, "hdfsGetDefaultBlockSize: %" PRId64 "\n", hdfsGetDefaultBlockSize(fs));
fprintf(stderr, "hdfsGetCapacity: %" PRId64 "\n", hdfsGetCapacity(fs));
fprintf(stderr, "hdfsGetUsed: %" PRId64 "\n", hdfsGetUsed(fs));
fileInfo = NULL;
if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
fprintf(stderr, "Name: %s, ", fileInfo->mName);
fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
fprintf(stderr, "BlockSize: %" PRId64 ", ", fileInfo->mBlockSize);
fprintf(stderr, "Size: %" PRId64 ", ", fileInfo->mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod));
fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
permission_disp(fileInfo->mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
hdfsFreeFileInfo(fileInfo, 1);
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
}
fileList = 0;
fileList = hdfsListDirectory(fs, newDirectory, &numEntries);
if (!(fileList == NULL && numEntries == 0 && !errno)) {
fprintf(stderr, "waah! hdfsListDirectory for empty %s - FAILED!\n", newDirectory);
totalResult++;
} else {
fprintf(stderr, "hdfsListDirectory for empty %s - SUCCESS!\n", newDirectory);
}
fileList = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
fprintf(stderr, "BlockSize: %" PRId64 ", ", fileList[i].mBlockSize);
fprintf(stderr, "Size: %" PRId64 ", ", fileList[i].mSize);
fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
permission_disp(fileList[i].mPermissions, permissions);
fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
}
hdfsFreeFileInfo(fileList, numEntries);
} else {
if (errno) {
totalResult++;
fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
} else {
fprintf(stderr, "Empty directory!\n");
}
}
hosts = hdfsGetHosts(fs, srcPath, 0, 1);
if(hosts) {
fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
i=0;
while(hosts[i]) {
j = 0;
while(hosts[i][j]) {
fprintf(stderr,
"\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
++j;
}
++i;
}
} else {
totalResult++;
fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
}
// setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
// chown write
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// chmod write
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
sleep(2);
newMtime = time(NULL);
newAtime = time(NULL);
// utime write
fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// chown/chmod/utime read
finfo = hdfsGetPathInfo(fs, writePath);
fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner))) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// will later use /tmp/ as a different user so enable it
fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr,"newMTime=%ld\n",newMtime);
fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);
fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
// No easy way to turn on access times from hdfs_test right now
// fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) != 0 ? "Failed!" : "Success!"));
// totalResult += result;
hdfsFreeFileInfo(finfo, 1);
// Clean up
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath, 1)) != 0 ? "Failed!" : "Success!"));
totalResult += result;
fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) != 0 ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
}
{
// TEST APPENDS
// CREATE
appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY, 0, 0, 0);
if(!appendFile) {
fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
buffer3 = "Hello,";
num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3,
(tSize)strlen(buffer3));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, appendFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", appendPath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", appendPath);
hdfsCloseFile(fs, appendFile);
// RE-OPEN
appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY|O_APPEND, 0, 0, 0);
if(!appendFile) {
fprintf(stderr, "Failed to open %s for writing!\n", appendPath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", appendPath);
buffer3 = " World";
num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3,
(tSize)(strlen(buffer3) + 1));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, appendFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", appendPath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", appendPath);
hdfsCloseFile(fs, appendFile);
// CHECK size
finfo = hdfsGetPathInfo(fs, appendPath);
fprintf(stderr, "fileinfo->mSize: == total %s\n", ((result = (finfo->mSize == (tOffset)(strlen("Hello, World") + 1))) == 1 ? "Success!" : "Failed!"));
totalResult += (result ? 0 : 1);
// READ and check data
readFile = hdfsOpenFile(fs, appendPath, O_RDONLY, 0, 0, 0);
if (!readFile) {
fprintf(stderr, "Failed to open %s for reading!\n", appendPath);
exit(-1);
}
num_read_bytes = hdfsRead(fs, readFile, (void*)rdbuffer, sizeof(rdbuffer));
fprintf(stderr, "Read following %d bytes:\n%s\n",
num_read_bytes, rdbuffer);
fprintf(stderr, "read == Hello, World %s\n", ((result = (strcmp(rdbuffer, "Hello, World"))) == 0 ? "Success!" : "Failed!"));
hdfsCloseFile(fs, readFile);
// DONE test appends
}
totalResult += (hdfsDisconnect(fs) != 0);
{
//
// Now test as connecting as a specific user
// This is only meant to test that we connected as that user, not to test
// the actual fs user capabilities. Thus just create a file and read
// the owner is correct.
fs = hdfsConnectAsUserNewInstance("default", 0, tuser);
if(!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
exit(-1);
}
userFile = hdfsOpenFile(fs, userPath, O_WRONLY|O_CREAT, 0, 0, 0);
if(!userFile) {
fprintf(stderr, "Failed to open %s for writing!\n", userPath);
exit(-1);
}
fprintf(stderr, "Opened %s for writing successfully...\n", userPath);
num_written_bytes = hdfsWrite(fs, userFile, (void*)fileContents,
(tSize)(strlen(fileContents)+1));
fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
if (hdfsFlush(fs, userFile)) {
fprintf(stderr, "Failed to 'flush' %s\n", userPath);
exit(-1);
}
fprintf(stderr, "Flushed %s successfully!\n", userPath);
hdfsCloseFile(fs, userFile);
finfo = hdfsGetPathInfo(fs, userPath);
fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser))) != 0 ? "Failed!" : "Success!"));
totalResult += result;
}
totalResult += (hdfsDisconnect(fs) != 0);
if (totalResult != 0) {
return -1;
} else {
return 0;
}
}
