ssize_t HdfsFile::pread(void *ptr, off64_t offset, size_t count) { boost::mutex::scoped_lock lock(m_mutex); tSize ret = hdfsPread(m_fs, m_file, offset, ptr, count); // this is an observed case when trying to read from an open file handle // that is now stale (because of rewrite/whatever). If we see this, // assume that the file has likely been rewritten and a subsequent attempt // to read will be successful if we reopen the file. tSize retryCount = 0; while (ret != (tSize) count) { //if (ret != count) // IDBLogger::logRW("pread_retry", m_fname, this, offset, count, ret); if (retryCount >= 50) break; usleep(1000); reopen(); ret = hdfsPread(m_fs, m_file, offset, ptr, count); retryCount++; } if( IDBLogger::isEnabled() ) IDBLogger::logRW("pread", m_fname, this, offset, count, ret); return ret; }
qioerr hdfs_preadv (void* file, const struct iovec *vector, int count, off_t offset, ssize_t* num_read_out, void* fs) { ssize_t got; ssize_t got_total; qioerr err_out = 0; int i; STARTING_SLOW_SYSCALL; err_out = 0; got_total = 0; for(i = 0; i < count; i++) { got = hdfsPread(to_hdfs_fs(fs)->hfs, to_hdfs_file(file)->file, offset + got_total, (void*)vector[i].iov_base, vector[i].iov_len); if( got != -1 ) { got_total += got; } else { err_out = qio_mkerror_errno(); break; } if(got != (ssize_t)vector[i].iov_len ) { break; } } if( err_out == 0 && got_total == 0 && sys_iov_total_bytes(vector, count) != 0 ) err_out = qio_int_to_err(EEOF); *num_read_out = got_total; DONE_SLOW_SYSCALL; return err_out; }
int BlockManager::loadFromHdfs(const ChunkID& chunk_id, void* const &desc,const unsigned & length){ lock.acquire(); int ret; int offset=chunk_id.chunk_off; hdfsFS fs=hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port); hdfsFile readFile=hdfsOpenFile(fs,chunk_id.partition_id.getPathAndName().c_str(),O_RDONLY,0,0,0); hdfsFileInfo *hdfsfile=hdfsGetPathInfo(fs,chunk_id.partition_id.getPathAndName().c_str());// to be refined after communicating with Zhang Lei if(!readFile){ logging_->elog("Fail to open file [%s].Reason:%s",chunk_id.partition_id.getPathAndName().c_str(),strerror(errno)); hdfsDisconnect(fs); lock.release(); return -1; } else{ logging_->log("file [%s] is opened for offset[%d]\n",chunk_id.partition_id.getPathAndName().c_str(),offset); } long int start_pos=CHUNK_SIZE*offset; if(start_pos<hdfsfile->mSize){ ret=hdfsPread(fs,readFile,start_pos,desc,length); }else{ lock.release(); ret= -1; } hdfsCloseFile(fs,readFile); hdfsDisconnect(fs); lock.release(); return ret; }
// 这个函数返回一个构造好的chunkid和每个chunk的指针 // 这个里面的chunkId肯定是要在blockManager注册然后汇报信息的 // put的话也是会这样的,可以将这个函数中调用put然后统一汇报信息的接口 ChunkInfo BlockManager::loadFromHdfs(string file_name){ // 由此函数得到的<blockId,指针> ChunkInfo ci; string file_name_former,file_name_latter; unsigned pos=file_name.rfind("$"); file_name_former=file_name.substr(0,pos); file_name_latter=file_name.substr(pos+1,file_name.length()); int offset=atoi(file_name_latter.c_str()); hdfsFS fs=hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port); hdfsFile readFile=hdfsOpenFile(fs,file_name_former.c_str(),O_RDONLY,0,0,0); hdfsFileInfo *hdfsfile=hdfsGetPathInfo(fs,file_name_former.c_str()); if(!readFile){ cout<<"open file error"<<endl; } unsigned length=0; length=length+CHUNK_SIZE*offset; if(length<hdfsfile->mSize){ void *rt=malloc(CHUNK_SIZE); //newmalloc tSize bytes_num=hdfsPread(fs,readFile,length,rt,CHUNK_SIZE); ostringstream chunkid; chunkid<<file_name.c_str()<<"$"<<offset; // ci.chunkId=chunkid.gestr().c_str(); ci.hook=rt; }else{ ostringstream chunkid; chunkid<<file_name.c_str()<<"$"<<offset; // ci.chunkId=chunkid.str().c_str(); ci.hook=0; } hdfsCloseFile(fs,readFile); hdfsDisconnect(fs); return ci; }
int hdfs_file_pread(struct back_storage *storage, bs_file_t file, \ const char *read_buff, uint32_t read_len, uint64_t pos){ //HLOG_DEBUG("hdfs -- enter func %s", __func__); hdfsFile hfile = (hdfsFile)file; //HLOG_DEBUG("hdfs -- leave func %s", __func__); return hdfsPread((hdfsFS)storage->fs_handler, \ hfile, pos, (void *) read_buff, read_len); }
/** * hdfsRead - Read data from an open file. * * @param fs The configured filesystem handle. * @param file The file handle. * @param buffer The buffer to copy read bytes into. * @param length The length of the buffer. * @return Returns the number of bytes actually read, possibly less * than than length; -1 on error. */ tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length) { struct hdfsFile_internal *f = file; tSize res; res = hdfsPread(fs, file, f->fi_offset, buffer, length); if (res != -1) f->fi_offset += length; return res; }
bool HDFSChunkReaderIterator::nextBlock(BlockStreamBase*& block){ if(cur_block_>=number_of_blocks_){ lock_.acquire(); return false; } tSize bytes_num=hdfsPread(fs_,hdfs_fd_,cur_block_*block_size_,block_buffer_->getBlock(),CHUNK_SIZE); if(bytes_num==block_size_){ printf("cur block=%d\n",cur_block_); cur_block_++; block->constructFromBlock(*block_buffer_); lock_.release(); return true; } else{ cur_block_++; lock_.release(); return false; } }
ssize_t HdfsFile::real_pread(void *ptr, off64_t offset, size_t count) { int savedErrno = -1; size_t progress = 0; uint8_t *ptr8 = (uint8_t *) ptr; tSize ret = 0; // this is an observed case when trying to read from an open file handle // that is now stale (because of rewrite/whatever). If we see this, // assume that the file has likely been rewritten and a subsequent attempt // to read will be successful if we reopen the file. /* Revised s.t. progress is not counted against the number of retry attempts. */ tSize retryCount = 0; while (progress < count && retryCount < 100) { ret = hdfsPread(m_fs, m_file, offset + progress, &ptr8[progress], count - progress); savedErrno = errno; if (ret > 0) { // making progress, reset the retry counter retryCount = 0; progress += ret; } else { // an error or EOF // cout << "retrying... progress = " << progress << " count = " << count << // " filename = " << name() << endl; if (retryCount < 10) usleep(1000); else usleep(200000); reopen(); // keep retrying, regardless of an error on reopen retryCount++; } } if( IDBLogger::isEnabled() ) IDBLogger::logRW("pread", m_fname, this, offset, count, ret); errno = savedErrno; return progress; }
int main(int argc, char **argv) { hdfsFS fs = hdfsConnect("default", 0); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs!\n"); exit(-1); } hdfsFS lfs = hdfsConnect(NULL, 0); if(!lfs) { fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n"); exit(-1); } const char* writePath = "/tmp/testfile.txt"; { //Write tests hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", writePath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", writePath); char* buffer = "Hello, World!"; tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); tOffset currentPos = -1; if ((currentPos = hdfsTell(fs, writeFile)) == -1) { fprintf(stderr, "Failed to get current file position correctly! Got %ld!\n", currentPos); exit(-1); } fprintf(stderr, "Current position: %ld\n", currentPos); if (hdfsFlush(fs, writeFile)) { fprintf(stderr, "Failed to 'flush' %s\n", writePath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", writePath); hdfsCloseFile(fs, writeFile); } { //Read tests const char* readPath = "/tmp/testfile.txt"; int exists = hdfsExists(fs, readPath); if (exists) { fprintf(stderr, "Failed to validate existence of %s\n", readPath); exit(-1); } hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for reading!\n", readPath); exit(-1); } fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile)); tOffset seekPos = 1; if(hdfsSeek(fs, readFile, seekPos)) { fprintf(stderr, "Failed to seek %s for reading!\n", readPath); exit(-1); } tOffset currentPos = -1; if((currentPos = hdfsTell(fs, readFile)) != seekPos) { fprintf(stderr, "Failed to get current file position correctly! Got %ld!\n", currentPos); exit(-1); } fprintf(stderr, "Current position: %ld\n", currentPos); static char buffer[32]; tSize num_read_bytes = hdfsRead(fs, readFile, (void*)buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); hdfsCloseFile(fs, readFile); } int totalResult = 0; int result = 0; { //Generic file-system operations const char* srcPath = "/tmp/testfile.txt"; const char* dstPath = "/tmp/testfile2.txt"; fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; const char* slashTmp = "/tmp"; const char* newDirectory = "/tmp/newdir"; fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) ? "Failed!" : "Success!")); totalResult += result; char buffer[256]; const char *resp; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!")); totalResult += (resp ? 0 : 1); fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!")); totalResult += (resp ? 0 : 1); fprintf(stderr, "hdfsGetDefaultBlockSize: %ld\n", hdfsGetDefaultBlockSize(fs)); fprintf(stderr, "hdfsGetCapacity: %ld\n", hdfsGetCapacity(fs)); fprintf(stderr, "hdfsGetUsed: %ld\n", hdfsGetUsed(fs)); hdfsFileInfo *fileInfo = NULL; if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) { fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n"); fprintf(stderr, "Name: %s, ", fileInfo->mName); fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind)); fprintf(stderr, "Replication: %d, ", fileInfo->mReplication); fprintf(stderr, "BlockSize: %ld, ", fileInfo->mBlockSize); fprintf(stderr, "Size: %ld, ", fileInfo->mSize); fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod)); fprintf(stderr, "Owner: %s, ", fileInfo->mOwner); fprintf(stderr, "Group: %s, ", fileInfo->mGroup); char permissions[10]; permission_disp(fileInfo->mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions); hdfsFreeFileInfo(fileInfo, 1); } else { totalResult++; fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp); } hdfsFileInfo *fileList = 0; int numEntries = 0; if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) { int i = 0; for(i=0; i < numEntries; ++i) { fprintf(stderr, "Name: %s, ", fileList[i].mName); fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind); fprintf(stderr, "Replication: %d, ", fileList[i].mReplication); fprintf(stderr, "BlockSize: %ld, ", fileList[i].mBlockSize); fprintf(stderr, "Size: %ld, ", fileList[i].mSize); fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod)); fprintf(stderr, "Owner: %s, ", fileList[i].mOwner); fprintf(stderr, "Group: %s, ", fileList[i].mGroup); char permissions[10]; permission_disp(fileList[i].mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions); } hdfsFreeFileInfo(fileList, numEntries); } else { if (errno) { totalResult++; fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n"); } else { fprintf(stderr, "Empty directory!\n"); } } char*** hosts = hdfsGetHosts(fs, srcPath, 0, 1); if(hosts) { fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n"); int i=0; while(hosts[i]) { int j = 0; while(hosts[i][j]) { fprintf(stderr, "\thosts[%d][%d] - %s\n", i, j, hosts[i][j]); ++j; } ++i; } } else { totalResult++; fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n"); } char *newOwner = "root"; // setting tmp dir to 777 so later when connectAsUser nobody, we can write to it short newPerm = 0666; // chown write fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) ? "Failed!" : "Success!")); totalResult += result; // chmod write fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) ? "Failed!" : "Success!")); totalResult += result; sleep(2); tTime newMtime = time(NULL); tTime newAtime = time(NULL); // utime write fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) ? "Failed!" : "Success!")); totalResult += result; // chown/chmod/utime read hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath); fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!")); totalResult += result; // will later use /tmp/ as a different user so enable it fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr,"newMTime=%ld\n",newMtime); fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod); fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) ? "Failed!" : "Success!")); totalResult += result; // No easy way to turn on access times from hdfs_test right now // fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) ? "Failed!" : "Success!")); // totalResult += result; hdfsFreeFileInfo(finfo, 1); // Clean up fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!")); totalResult += (result ? 0 : 1); } totalResult += (hdfsDisconnect(fs) != 0); { // // Now test as connecting as a specific user // This is only meant to test that we connected as that user, not to test // the actual fs user capabilities. Thus just create a file and read // the owner is correct. const char *tuser = "******"; const char* writePath = "/tmp/usertestfile.txt"; const char **groups = (const char**)malloc(sizeof(char*)* 2); groups[0] = "users"; groups[1] = "nobody"; fs = hdfsConnectAsUser("default", 0, tuser, groups, 2); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser); exit(-1); } hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", writePath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", writePath); char* buffer = "Hello, World!"; tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); if (hdfsFlush(fs, writeFile)) { fprintf(stderr, "Failed to 'flush' %s\n", writePath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", writePath); hdfsCloseFile(fs, writeFile); hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath); fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!")); totalResult += result; } totalResult += (hdfsDisconnect(fs) != 0); if (totalResult != 0) { return -1; } else { return 0; } }
NABoolean HHDFSFileStats::populate(hdfsFS fs, hdfsFileInfo *fileInfo, Int32& samples, NABoolean doEstimation, char recordTerminator, NABoolean isSequenceFile) { NABoolean result = TRUE; // copy fields from fileInfo fileName_ = fileInfo->mName; replication_ = (Int32) fileInfo->mReplication; totalSize_ = (Int64) fileInfo->mSize; blockSize_ = (Int64) fileInfo->mBlockSize; modificationTS_ = fileInfo->mLastMod; numFiles_ = 1; isSequenceFile_ = isSequenceFile; Int64 sampleBufferSize = MINOF(blockSize_, 65536); NABoolean sortHosts = (CmpCommon::getDefault(HIVE_SORT_HDFS_HOSTS) == DF_ON); sampleBufferSize = MINOF(sampleBufferSize,totalSize_/10); if (doEstimation && sampleBufferSize > 100) { // // Open the hdfs file to estimate record length. Read one block at // a time searching for <s> instances of record separators. Stop reading // when either <s> instances have been found or a partial number of // instances have and we have exhausted all data content in the block. // We will keep reading if the current block does not contain // any instance of the record separator. // hdfsFile file = hdfsOpenFile(fs, fileInfo->mName, O_RDONLY, sampleBufferSize, // buffer size 0, // replication, take the default size fileInfo->mBlockSize // blocksize ); if ( file != NULL ) { tOffset offset = 0; tSize bufLen = sampleBufferSize; char* buffer = new (heap_) char[bufLen+1]; buffer[bufLen] = 0; // extra null at the end to protect strchr() // to run over the buffer. NABoolean sampleDone = FALSE; Int32 totalSamples = 10; Int32 totalLen = 0; while (!sampleDone) { tSize szRead = hdfsPread(fs, file, offset, buffer, bufLen); char* pos = NULL; if ( szRead > 0 ) { //if (isSequenceFile && offset==0 && memcmp(buffer, "SEQ6", 4) == 0) // isSequenceFile_ = TRUE; char* start = buffer; for (Int32 i=0; i<totalSamples; i++ ) { if ( (pos=strchr(start, recordTerminator)) ) { totalLen += pos - start + 1 + offset; samples++; start = pos+1; if ( start > buffer + bufLen ) { sampleDone = TRUE; break; } } else break; } if ( samples > 0 ) break; else offset += bufLen; } else break; // fail to read any bytes. Bail out. } NADELETEBASIC(buffer, heap_); if ( samples > 0 ) { sampledBytes_ += totalLen; sampledRows_ += samples; } hdfsCloseFile(fs, file); } else { // can not do hdfs open on the file. Assume the file is empty. } } if (blockSize_) { numBlocks_ = totalSize_ / blockSize_; if (totalSize_ % blockSize_ > 0) numBlocks_++; // partial block at the end } else { CMPASSERT(blockSize_); // TBD:DIAGS result = FALSE; } if ( totalSize_ > 0 ) { blockHosts_ = new(heap_) HostId[replication_*numBlocks_]; // walk through blocks and record their locations tOffset o = 0; Int64 blockNum; for (blockNum=0; blockNum < numBlocks_ && result; blockNum++) { char*** blockHostNames = hdfsGetHosts(fs, fileInfo->mName, o, fileInfo->mBlockSize); o += blockSize_; if (blockHostNames == NULL) { CMPASSERT(blockHostNames); // TBD:DIAGS result = FALSE; } else { char **h = *blockHostNames; HostId hostId; for (Int32 r=0; r<replication_; r++) { if (h[r]) hostId = HHDFSMasterHostList::getHostNum(h[r]); else hostId = HHDFSMasterHostList::InvalidHostId; blockHosts_[r*numBlocks_+blockNum] = hostId; } if (sortHosts) sortHostArray(blockHosts_, (Int32) numBlocks_, replication_, getFileName()); } hdfsFreeHosts(blockHostNames); } } return result; }
int main(int argc, char **argv) { const char *writePath = "/tmp/testfile.txt"; const char *fileContents = "Hello, World!"; const char *readPath = "/tmp/testfile.txt"; const char *srcPath = "/tmp/testfile.txt"; const char *dstPath = "/tmp/testfile2.txt"; const char *slashTmp = "/tmp"; const char *newDirectory = "/tmp/newdir"; const char *newOwner = "root"; const char *tuser = "******"; const char *appendPath = "/tmp/appends"; const char *userPath = "/tmp/usertestfile.txt"; char buffer[32], buffer2[256], rdbuffer[32]; tSize num_written_bytes, num_read_bytes; hdfsFS fs, lfs; hdfsFile writeFile, readFile, localFile, appendFile, userFile; tOffset currentPos, seekPos; int exists, totalResult, result, numEntries, i, j; const char *resp; hdfsFileInfo *fileInfo, *fileList, *finfo; char *buffer3; char permissions[10]; char ***hosts; short newPerm = 0666; tTime newMtime, newAtime; fs = hdfsConnectNewInstance("default", 0); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs!\n"); exit(-1); } lfs = hdfsConnectNewInstance(NULL, 0); if(!lfs) { fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n"); exit(-1); } { //Write tests writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", writePath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", writePath); num_written_bytes = hdfsWrite(fs, writeFile, (void*)fileContents, (tSize)(strlen(fileContents)+1)); if (num_written_bytes != strlen(fileContents) + 1) { fprintf(stderr, "Failed to write correct number of bytes - expected %d, got %d\n", (int)(strlen(fileContents) + 1), (int)num_written_bytes); exit(-1); } fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); currentPos = -1; if ((currentPos = hdfsTell(fs, writeFile)) == -1) { fprintf(stderr, "Failed to get current file position correctly! Got %" PRId64 "!\n", currentPos); exit(-1); } fprintf(stderr, "Current position: %" PRId64 "\n", currentPos); if (hdfsFlush(fs, writeFile)) { fprintf(stderr, "Failed to 'flush' %s\n", writePath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", writePath); if (hdfsHFlush(fs, writeFile)) { fprintf(stderr, "Failed to 'hflush' %s\n", writePath); exit(-1); } fprintf(stderr, "HFlushed %s successfully!\n", writePath); hdfsCloseFile(fs, writeFile); } { //Read tests exists = hdfsExists(fs, readPath); if (exists) { fprintf(stderr, "Failed to validate existence of %s\n", readPath); exit(-1); } readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for reading!\n", readPath); exit(-1); } if (!hdfsFileIsOpenForRead(readFile)) { fprintf(stderr, "hdfsFileIsOpenForRead: we just opened a file " "with O_RDONLY, and it did not show up as 'open for " "read'\n"); exit(-1); } fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile)); seekPos = 1; if(hdfsSeek(fs, readFile, seekPos)) { fprintf(stderr, "Failed to seek %s for reading!\n", readPath); exit(-1); } currentPos = -1; if((currentPos = hdfsTell(fs, readFile)) != seekPos) { fprintf(stderr, "Failed to get current file position correctly! Got %" PRId64 "!\n", currentPos); exit(-1); } fprintf(stderr, "Current position: %" PRId64 "\n", currentPos); if (!hdfsFileUsesDirectRead(readFile)) { fprintf(stderr, "Direct read support incorrectly not detected " "for HDFS filesystem\n"); exit(-1); } fprintf(stderr, "Direct read support detected for HDFS\n"); // Test the direct read path if(hdfsSeek(fs, readFile, 0)) { fprintf(stderr, "Failed to seek %s for reading!\n", readPath); exit(-1); } memset(buffer, 0, sizeof(buffer)); num_read_bytes = hdfsRead(fs, readFile, (void*)buffer, sizeof(buffer)); if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) { fprintf(stderr, "Failed to read (direct). Expected %s but got %s (%d bytes)\n", fileContents, buffer, num_read_bytes); exit(-1); } fprintf(stderr, "Read (direct) following %d bytes:\n%s\n", num_read_bytes, buffer); if (hdfsSeek(fs, readFile, 0L)) { fprintf(stderr, "Failed to seek to file start!\n"); exit(-1); } // Disable the direct read path so that we really go through the slow // read path hdfsFileDisableDirectRead(readFile); num_read_bytes = hdfsRead(fs, readFile, (void*)buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); memset(buffer, 0, strlen(fileContents + 1)); num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); hdfsCloseFile(fs, readFile); // Test correct behaviour for unsupported filesystems localFile = hdfsOpenFile(lfs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!localFile) { fprintf(stderr, "Failed to open %s for writing!\n", writePath); exit(-1); } num_written_bytes = hdfsWrite(lfs, localFile, (void*)fileContents, (tSize)(strlen(fileContents) + 1)); hdfsCloseFile(lfs, localFile); localFile = hdfsOpenFile(lfs, writePath, O_RDONLY, 0, 0, 0); if (hdfsFileUsesDirectRead(localFile)) { fprintf(stderr, "Direct read support incorrectly detected for local " "filesystem\n"); exit(-1); } hdfsCloseFile(lfs, localFile); } totalResult = 0; result = 0; { //Generic file-system operations fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!")); totalResult += (resp ? 0 : 1); fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer2, sizeof(buffer2))) != 0 ? buffer2 : "Failed!")); totalResult += (resp ? 0 : 1); fprintf(stderr, "hdfsGetDefaultBlockSize: %" PRId64 "\n", hdfsGetDefaultBlockSize(fs)); fprintf(stderr, "hdfsGetCapacity: %" PRId64 "\n", hdfsGetCapacity(fs)); fprintf(stderr, "hdfsGetUsed: %" PRId64 "\n", hdfsGetUsed(fs)); fileInfo = NULL; if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) { fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n"); fprintf(stderr, "Name: %s, ", fileInfo->mName); fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind)); fprintf(stderr, "Replication: %d, ", fileInfo->mReplication); fprintf(stderr, "BlockSize: %" PRId64 ", ", fileInfo->mBlockSize); fprintf(stderr, "Size: %" PRId64 ", ", fileInfo->mSize); fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod)); fprintf(stderr, "Owner: %s, ", fileInfo->mOwner); fprintf(stderr, "Group: %s, ", fileInfo->mGroup); permission_disp(fileInfo->mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions); hdfsFreeFileInfo(fileInfo, 1); } else { totalResult++; fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp); } fileList = 0; fileList = hdfsListDirectory(fs, newDirectory, &numEntries); if (!(fileList == NULL && numEntries == 0 && !errno)) { fprintf(stderr, "waah! hdfsListDirectory for empty %s - FAILED!\n", newDirectory); totalResult++; } else { fprintf(stderr, "hdfsListDirectory for empty %s - SUCCESS!\n", newDirectory); } fileList = 0; if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) { for(i=0; i < numEntries; ++i) { fprintf(stderr, "Name: %s, ", fileList[i].mName); fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind); fprintf(stderr, "Replication: %d, ", fileList[i].mReplication); fprintf(stderr, "BlockSize: %" PRId64 ", ", fileList[i].mBlockSize); fprintf(stderr, "Size: %" PRId64 ", ", fileList[i].mSize); fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod)); fprintf(stderr, "Owner: %s, ", fileList[i].mOwner); fprintf(stderr, "Group: %s, ", fileList[i].mGroup); permission_disp(fileList[i].mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions); } hdfsFreeFileInfo(fileList, numEntries); } else { if (errno) { totalResult++; fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n"); } else { fprintf(stderr, "Empty directory!\n"); } } hosts = hdfsGetHosts(fs, srcPath, 0, 1); if(hosts) { fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n"); i=0; while(hosts[i]) { j = 0; while(hosts[i][j]) { fprintf(stderr, "\thosts[%d][%d] - %s\n", i, j, hosts[i][j]); ++j; } ++i; } } else { totalResult++; fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n"); } // setting tmp dir to 777 so later when connectAsUser nobody, we can write to it // chown write fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) != 0 ? "Failed!" : "Success!")); totalResult += result; // chmod write fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) != 0 ? "Failed!" : "Success!")); totalResult += result; sleep(2); newMtime = time(NULL); newAtime = time(NULL); // utime write fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) != 0 ? "Failed!" : "Success!")); totalResult += result; // chown/chmod/utime read finfo = hdfsGetPathInfo(fs, writePath); fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner))) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) != 0 ? "Failed!" : "Success!")); totalResult += result; // will later use /tmp/ as a different user so enable it fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr,"newMTime=%ld\n",newMtime); fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod); fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) != 0 ? "Failed!" : "Success!")); totalResult += result; // No easy way to turn on access times from hdfs_test right now // fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) != 0 ? "Failed!" : "Success!")); // totalResult += result; hdfsFreeFileInfo(finfo, 1); // Clean up fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory, 1)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath, 1)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath, 1)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath, 1)) != 0 ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) != 0 ? "Success!" : "Failed!")); totalResult += (result ? 0 : 1); } { // TEST APPENDS // CREATE appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY, 0, 0, 0); if(!appendFile) { fprintf(stderr, "Failed to open %s for writing!\n", appendPath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", appendPath); buffer3 = "Hello,"; num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3, (tSize)strlen(buffer3)); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); if (hdfsFlush(fs, appendFile)) { fprintf(stderr, "Failed to 'flush' %s\n", appendPath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", appendPath); hdfsCloseFile(fs, appendFile); // RE-OPEN appendFile = hdfsOpenFile(fs, appendPath, O_WRONLY|O_APPEND, 0, 0, 0); if(!appendFile) { fprintf(stderr, "Failed to open %s for writing!\n", appendPath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", appendPath); buffer3 = " World"; num_written_bytes = hdfsWrite(fs, appendFile, (void*)buffer3, (tSize)(strlen(buffer3) + 1)); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); if (hdfsFlush(fs, appendFile)) { fprintf(stderr, "Failed to 'flush' %s\n", appendPath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", appendPath); hdfsCloseFile(fs, appendFile); // CHECK size finfo = hdfsGetPathInfo(fs, appendPath); fprintf(stderr, "fileinfo->mSize: == total %s\n", ((result = (finfo->mSize == (tOffset)(strlen("Hello, World") + 1))) == 1 ? "Success!" : "Failed!")); totalResult += (result ? 0 : 1); // READ and check data readFile = hdfsOpenFile(fs, appendPath, O_RDONLY, 0, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for reading!\n", appendPath); exit(-1); } num_read_bytes = hdfsRead(fs, readFile, (void*)rdbuffer, sizeof(rdbuffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, rdbuffer); fprintf(stderr, "read == Hello, World %s\n", ((result = (strcmp(rdbuffer, "Hello, World"))) == 0 ? "Success!" : "Failed!")); hdfsCloseFile(fs, readFile); // DONE test appends } totalResult += (hdfsDisconnect(fs) != 0); { // // Now test as connecting as a specific user // This is only meant to test that we connected as that user, not to test // the actual fs user capabilities. Thus just create a file and read // the owner is correct. fs = hdfsConnectAsUserNewInstance("default", 0, tuser); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser); exit(-1); } userFile = hdfsOpenFile(fs, userPath, O_WRONLY|O_CREAT, 0, 0, 0); if(!userFile) { fprintf(stderr, "Failed to open %s for writing!\n", userPath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", userPath); num_written_bytes = hdfsWrite(fs, userFile, (void*)fileContents, (tSize)(strlen(fileContents)+1)); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); if (hdfsFlush(fs, userFile)) { fprintf(stderr, "Failed to 'flush' %s\n", userPath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", userPath); hdfsCloseFile(fs, userFile); finfo = hdfsGetPathInfo(fs, userPath); fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser))) != 0 ? "Failed!" : "Success!")); totalResult += result; } totalResult += (hdfsDisconnect(fs) != 0); if (totalResult != 0) { return -1; } else { return 0; } }
int main(int argc, char **argv) { char buffer[32]; tSize num_written_bytes; const char* slashTmp = "/tmp"; int nnPort; char *rwTemplate, *rwTemplate2, *newDirTemplate, *appendTemplate, *userTemplate, *rwPath = NULL; const char* fileContents = "Hello, World!"; const char* nnHost = NULL; if (argc != 2) { fprintf(stderr, "usage: test_libwebhdfs_ops <username>\n"); exit(1); } struct NativeMiniDfsConf conf = { .doFormat = 1, .webhdfsEnabled = 1, .namenodeHttpPort = 50070, }; cluster = nmdCreate(&conf); if (!cluster) { fprintf(stderr, "Failed to create the NativeMiniDfsCluster.\n"); exit(1); } if (nmdWaitClusterUp(cluster)) { fprintf(stderr, "Error when waiting for cluster to be ready.\n"); exit(1); } if (nmdGetNameNodeHttpAddress(cluster, &nnPort, &nnHost)) { fprintf(stderr, "Error when retrieving namenode host address.\n"); exit(1); } hdfsFS fs = hdfsConnectAsUserNewInstance(nnHost, nnPort, argv[1]); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs!\n"); exit(-1); } { // Write tests rwTemplate = strdup("/tmp/helloWorldXXXXXX"); if (!rwTemplate) { fprintf(stderr, "Failed to create rwTemplate!\n"); exit(1); } rwPath = mktemp(rwTemplate); // hdfsOpenFile hdfsFile writeFile = hdfsOpenFile(fs, rwPath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", rwPath); exit(1); } fprintf(stderr, "Opened %s for writing successfully...\n", rwPath); // hdfsWrite num_written_bytes = hdfsWrite(fs, writeFile, (void*)fileContents, (int) strlen(fileContents) + 1); if (num_written_bytes != strlen(fileContents) + 1) { fprintf(stderr, "Failed to write correct number of bytes - " "expected %d, got %d\n", (int)(strlen(fileContents) + 1), (int) num_written_bytes); exit(1); } fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); // hdfsTell tOffset currentPos = -1; if ((currentPos = hdfsTell(fs, writeFile)) == -1) { fprintf(stderr, "Failed to get current file position correctly. Got %" PRId64 "!\n", currentPos); exit(1); } fprintf(stderr, "Current position: %" PRId64 "\n", currentPos); hdfsCloseFile(fs, writeFile); // Done test write } sleep(1); { //Read tests int available = 0, exists = 0; // hdfsExists exists = hdfsExists(fs, rwPath); if (exists) { fprintf(stderr, "Failed to validate existence of %s\n", rwPath); exists = hdfsExists(fs, rwPath); if (exists) { fprintf(stderr, "Still failed to validate existence of %s\n", rwPath); exit(1); } } hdfsFile readFile = hdfsOpenFile(fs, rwPath, O_RDONLY, 0, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for reading!\n", rwPath); exit(1); } if (!hdfsFileIsOpenForRead(readFile)) { fprintf(stderr, "hdfsFileIsOpenForRead: we just opened a file " "with O_RDONLY, and it did not show up as 'open for " "read'\n"); exit(1); } available = hdfsAvailable(fs, readFile); fprintf(stderr, "hdfsAvailable: %d\n", available); // hdfsSeek, hdfsTell tOffset seekPos = 1; if(hdfsSeek(fs, readFile, seekPos)) { fprintf(stderr, "Failed to seek %s for reading!\n", rwPath); exit(1); } tOffset currentPos = -1; if((currentPos = hdfsTell(fs, readFile)) != seekPos) { fprintf(stderr, "Failed to get current file position correctly! Got %" PRId64 "!\n", currentPos); exit(1); } fprintf(stderr, "Current position: %" PRId64 "\n", currentPos); if(hdfsSeek(fs, readFile, 0)) { fprintf(stderr, "Failed to seek %s for reading!\n", rwPath); exit(1); } // hdfsRead memset(buffer, 0, sizeof(buffer)); tSize num_read_bytes = hdfsRead(fs, readFile, buffer, sizeof(buffer)); if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) { fprintf(stderr, "Failed to read (direct). " "Expected %s but got %s (%d bytes)\n", fileContents, buffer, num_read_bytes); exit(1); } fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); if (hdfsSeek(fs, readFile, 0L)) { fprintf(stderr, "Failed to seek to file start!\n"); exit(1); } // hdfsPread memset(buffer, 0, strlen(fileContents + 1)); num_read_bytes = hdfsPread(fs, readFile, 0, buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); hdfsCloseFile(fs, readFile); // Done test read } int totalResult = 0; int result = 0; { //Generic file-system operations char *srcPath = rwPath; char buffer[256]; const char *resp; rwTemplate2 = strdup("/tmp/helloWorld2XXXXXX"); if (!rwTemplate2) { fprintf(stderr, "Failed to create rwTemplate2!\n"); exit(1); } char *dstPath = mktemp(rwTemplate2); newDirTemplate = strdup("/tmp/newdirXXXXXX"); if (!newDirTemplate) { fprintf(stderr, "Failed to create newDirTemplate!\n"); exit(1); } char *newDirectory = mktemp(newDirTemplate); // hdfsRename fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, rwPath, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsRename back: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!")); totalResult += result; // hdfsCreateDirectory fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!")); totalResult += result; // hdfsSetReplication fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 1)) ? "Failed!" : "Success!")); totalResult += result; // hdfsGetWorkingDirectory, hdfsSetWorkingDirectory fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!")); totalResult += (resp ? 0 : 1); const char* path[] = {"/foo", "/foo/bar", "foobar", "//foo/bar//foobar", "foo//bar", "foo/bar///", "/", "////"}; int i; for (i = 0; i < 8; i++) { fprintf(stderr, "hdfsSetWorkingDirectory: %s, %s\n", ((result = hdfsSetWorkingDirectory(fs, path[i])) ? "Failed!" : "Success!"), hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))); totalResult += result; } fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!")); totalResult += (resp ? 0 : 1); // hdfsGetPathInfo hdfsFileInfo *fileInfo = NULL; if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) { fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n"); fprintf(stderr, "Name: %s, ", fileInfo->mName); fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind)); fprintf(stderr, "Replication: %d, ", fileInfo->mReplication); fprintf(stderr, "BlockSize: %"PRId64", ", fileInfo->mBlockSize); fprintf(stderr, "Size: %"PRId64", ", fileInfo->mSize); fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod)); fprintf(stderr, "Owner: %s, ", fileInfo->mOwner); fprintf(stderr, "Group: %s, ", fileInfo->mGroup); char permissions[10]; permission_disp(fileInfo->mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions); hdfsFreeFileInfo(fileInfo, 1); } else { totalResult++; fprintf(stderr, "hdfsGetPathInfo for %s - FAILED!\n", slashTmp); } // hdfsListDirectory hdfsFileInfo *fileList = 0; int numEntries = 0; if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) { int i = 0; for(i=0; i < numEntries; ++i) { fprintf(stderr, "Name: %s, ", fileList[i].mName); fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind); fprintf(stderr, "Replication: %d, ", fileList[i].mReplication); fprintf(stderr, "BlockSize: %"PRId64", ", fileList[i].mBlockSize); fprintf(stderr, "Size: %"PRId64", ", fileList[i].mSize); fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod)); fprintf(stderr, "Owner: %s, ", fileList[i].mOwner); fprintf(stderr, "Group: %s, ", fileList[i].mGroup); char permissions[10]; permission_disp(fileList[i].mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions); } hdfsFreeFileInfo(fileList, numEntries); } else { if (errno) { totalResult++; fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n"); } else { fprintf(stderr, "Empty directory!\n"); } } char *newOwner = "root"; // Setting tmp dir to 777 so later when connectAsUser nobody, // we can write to it short newPerm = 0666; // hdfsChown fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, rwPath, NULL, "users")) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, rwPath, newOwner, NULL)) ? "Failed!" : "Success!")); totalResult += result; // hdfsChmod fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, rwPath, newPerm)) ? "Failed!" : "Success!")); totalResult += result; sleep(2); tTime newMtime = time(NULL); tTime newAtime = time(NULL); // utime write fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, rwPath, newMtime, newAtime)) ? "Failed!" : "Success!")); totalResult += result; // chown/chmod/utime read hdfsFileInfo *finfo = hdfsGetPathInfo(fs, rwPath); fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!")); totalResult += result; // will later use /tmp/ as a different user so enable it fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, slashTmp, 0777)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr,"newMTime=%ld\n",newMtime); fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod); fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime / 1000)) ? "Failed!" : "Success!")); totalResult += result; // Clean up hdfsFreeFileInfo(finfo, 1); fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory, 1)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath, 1)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!")); totalResult += (result ? 0 : 1); // Done test generic operations } { // Test Appends appendTemplate = strdup("/tmp/appendsXXXXXX"); if (!appendTemplate) { fprintf(stderr, "Failed to create appendTemplate!\n"); exit(1); } char *appendPath = mktemp(appendTemplate); const char* helloBuffer = "Hello,"; hdfsFile writeFile = NULL; // Create writeFile = hdfsOpenFile(fs, appendPath, O_WRONLY, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", appendPath); exit(1); } fprintf(stderr, "Opened %s for writing successfully...\n", appendPath); num_written_bytes = hdfsWrite(fs, writeFile, helloBuffer, (int) strlen(helloBuffer)); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); hdfsCloseFile(fs, writeFile); fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, appendPath, 1)) ? "Failed!" : "Success!")); totalResult += result; // Re-Open for Append writeFile = hdfsOpenFile(fs, appendPath, O_WRONLY | O_APPEND, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", appendPath); exit(1); } fprintf(stderr, "Opened %s for appending successfully...\n", appendPath); helloBuffer = " World"; num_written_bytes = hdfsWrite(fs, writeFile, helloBuffer, (int)strlen(helloBuffer) + 1); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); hdfsCloseFile(fs, writeFile); // Check size hdfsFileInfo *finfo = hdfsGetPathInfo(fs, appendPath); fprintf(stderr, "fileinfo->mSize: == total %s\n", ((result = (finfo->mSize == strlen("Hello, World") + 1)) ? "Success!" : "Failed!")); totalResult += (result ? 0 : 1); // Read and check data hdfsFile readFile = hdfsOpenFile(fs, appendPath, O_RDONLY, 0, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for reading!\n", appendPath); exit(1); } tSize num_read_bytes = hdfsRead(fs, readFile, buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); fprintf(stderr, "read == Hello, World %s\n", (result = (strcmp(buffer, "Hello, World") == 0)) ? "Success!" : "Failed!"); hdfsCloseFile(fs, readFile); // Cleanup fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, appendPath, 1)) ? "Failed!" : "Success!")); totalResult += result; // Done test appends } totalResult += (hdfsDisconnect(fs) != 0); { // // Now test as connecting as a specific user // This only meant to test that we connected as that user, not to test // the actual fs user capabilities. Thus just create a file and read // the owner is correct. const char *tuser = "******"; userTemplate = strdup("/tmp/usertestXXXXXX"); if (!userTemplate) { fprintf(stderr, "Failed to create userTemplate!\n"); exit(1); } char* userWritePath = mktemp(userTemplate); hdfsFile writeFile = NULL; fs = hdfsConnectAsUserNewInstance("default", 50070, tuser); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser); exit(1); } writeFile = hdfsOpenFile(fs, userWritePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", userWritePath); exit(1); } fprintf(stderr, "Opened %s for writing successfully...\n", userWritePath); num_written_bytes = hdfsWrite(fs, writeFile, fileContents, (int)strlen(fileContents) + 1); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); hdfsCloseFile(fs, writeFile); hdfsFileInfo *finfo = hdfsGetPathInfo(fs, userWritePath); if (finfo) { fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!")); } else { fprintf(stderr, "hdfsFileInfo returned by hdfsGetPathInfo is NULL\n"); result = -1; } totalResult += result; // Cleanup fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, userWritePath, 1)) ? "Failed!" : "Success!")); totalResult += result; // Done test specific user } totalResult += (hdfsDisconnect(fs) != 0); // Shutdown the native minidfscluster nmdShutdown(cluster); nmdFree(cluster); fprintf(stderr, "totalResult == %d\n", totalResult); if (totalResult != 0) { return -1; } else { return 0; } }
qioerr hdfs_preadv (void* file, const struct iovec *vector, int count, off_t offset, ssize_t* num_read_out, void* fs) { ssize_t got; ssize_t got_total; qioerr err_out = 0; int i; STARTING_SLOW_SYSCALL; #ifdef HDFS3 const hdfs_file orig_hfl = *to_hdfs_file(file); const hdfs_fs orig_hfs = *to_hdfs_fs(fs); hdfsFS hfs = hdfsConnect(orig_hfs.fs_name, orig_hfs.fs_port); hdfsFile hfl = hdfsOpenFile(hfs, orig_hfl.pathnm, O_RDONLY, 0, 0, 0); //assert connection CREATE_ERROR((hfs == NULL), err_out, ECONNREFUSED, "Unable to read HDFS file", error); if(hfl == NULL) { err_out = qio_mkerror_errno(); goto error; } #endif err_out = 0; got_total = 0; for(i = 0; i < count; i++) { #ifdef HDFS3 hdfsSeek(hfs, hfl, offset+got_total); got = hdfsRead(hfs, hfl, (void*)vector[i].iov_base, vector[i].iov_len); #else got = hdfsPread(to_hdfs_fs(fs)->hfs, to_hdfs_file(file)->file, offset + got_total, (void*)vector[i].iov_base, vector[i].iov_len); #endif if( got != -1 ) { got_total += got; } else { err_out = qio_mkerror_errno(); break; } if(got != (ssize_t)vector[i].iov_len ) { break; } } if( err_out == 0 && got_total == 0 && sys_iov_total_bytes(vector, count) != 0 ) err_out = qio_int_to_err(EEOF); *num_read_out = got_total; #ifdef HDFS3 got = hdfsCloseFile(hfs, hfl); if(got == -1) { err_out = qio_mkerror_errno(); } got = hdfsDisconnect(hfs); if(got == -1) { err_out = qio_mkerror_errno(); } #endif DONE_SLOW_SYSCALL; #ifdef HDFS3 error: #endif return err_out; }