bool MaprFileSystem::IsDirectory(const std::string& uri){ std::string path = GetUriPathOrDie(uri); std::string host = "default"; hdfsFS fs = hdfsConnect(host.c_str(), 0); // use default config file settings CHECK(fs); hdfsFileInfo* info = hdfsGetPathInfo(fs, path.c_str()); bool is_directory = false; if (info){ is_directory = (info->mKind == kObjectKindDirectory); hdfsFreeFileInfo(info,1); } else{ LOG(FATAL) << "uri does not exist: " << uri; } CHECK_EQ(hdfsDisconnect(fs), 0); return is_directory; }
void ClaimsHDFS::claimsRead(){ hdfsFS fs; hdfsFile fd; string filename="/home/casa/data/kmeans_data.txt"; fs=hdfsConnect("10.11.1.174",9000); fd=hdfsOpenFile(fs,filename.c_str(),O_RDONLY,0,0,0); if(!fd){ cout<<"failed to open hdfs file!!!"<<endl; } char array[72]; tSize bytes=hdfsRead(fs,fd,array,72); cout<<"string is: "<<array<<endl; hdfsCloseFile(fs,fd); hdfsDisconnect(fs); }
hdfsFS HdfsConnection::getConnection(std::string host, int port) { QString id = _makeId(host, port); if (_connections.contains(id)) { return _connections[id]; } else { hdfsFS fs = hdfsConnect(host.c_str(), port); if (fs == NULL) { throw std::ios_base::failure("Error connecting to HDFS."); } _connections[id] = fs; return fs; } }
static PyObject * pyhdfsFS_delete(char* filepath) { // Parse HDFS information char* hostport = (char *)malloc(strlen(filepath)+1); char* buf = NULL; char* path; char* portStr; char* host; int hdfsPort; int ret = 0; ret = sscanf(filepath, "hdfs://%s", hostport); host = strtok_r(hostport, ":", &buf); portStr = strtok_r(NULL, "/", &buf); ret = sscanf(portStr, "%d", &hdfsPort); hdfsFS fs = hdfsConnect(host, hdfsPort); if (!fs) { PyErr_SetString(exception, "Cannot connect to host"); return exception; } path = (char*) malloc(strlen(buf)+2); path[0] = '/'; memcpy(path+1,buf,strlen(buf)); path[strlen(buf)+1] = '\0'; if (hdfsExists(fs, path) == 0) { hdfsDelete(fs, path); } else { PyErr_SetString(exception, "Cannot delete file"); return NULL; } free(hostport); free(path); return Py_BuildValue("i",1); }
int dfsList(const char* path){ hdfsFS fs = hdfsConnect("default", 0); int i, entries; hdfsFileInfo *files, *head; /* Get the list info */ files = hdfsListDirectory(fs, path, &entries); if (!files){ perror("Get directory info"); exit(-1); } head = files; /* Print the info */ fprintf(stdout, "%s %-50s %-9s %s\n", "Kind", "Name", "Size", "Replicas"); for (i = 0; i < entries; i++){ const char* unit[] = {" B", "KB", "MB", "GB", "TB", "PB"}; double size = files->mSize; unsigned int u = 0; while (size > 1024){ u++; size /= 1024; } assert(u < 6); fprintf(stdout, "%4c %-50s %-7.2lf%s %8d\n", files->mKind, files->mName, size, unit[u], files->mReplication); files += 1; } /* List ends */ hdfsFreeFileInfo(head, entries); hdfsDisconnect(fs); return 0; }
int main(int argc, char **argv) { if (argc != 4) { fprintf(stderr, "Usage: hdfs_read <filename> <filesize> <buffersize>\n"); exit(-1); } hdfsFS fs = hdfsConnect("default", 0); if (!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs!\n"); exit(-1); } const char* rfile = argv[1]; tSize fileTotalSize = strtoul(argv[2], NULL, 10); tSize bufferSize = strtoul(argv[3], NULL, 10); hdfsFile readFile = hdfsOpenFile(fs, rfile, O_RDONLY, bufferSize, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for writing!\n", rfile); exit(-2); } // data to be written to the file char* buffer = malloc(sizeof(char) * bufferSize); if(buffer == NULL) { return -2; } // read from the file tSize curSize = bufferSize; for (; curSize == bufferSize;) { curSize = hdfsRead(fs, readFile, (void*)buffer, curSize); } free(buffer); hdfsCloseFile(fs, readFile); hdfsDisconnect(fs); return 0; }
qioerr hdfs_connect(void** fs_out, const char* pathname, int port) { qioerr err_out = 0; hdfsFS fs; hdfs_fs* ret = (hdfs_fs*)qio_calloc(sizeof(hdfs_fs), 1); STARTING_SLOW_SYSCALL; fs = hdfsConnect(pathname, port); CREATE_ERROR((fs == NULL), err_out, ECONNREFUSED,"Unable to connect HDFS", error); DONE_SLOW_SYSCALL; ret->hfs = fs; DO_INIT_REFCNT(ret); ret->fs_name = pathname; ret->fs_port = port; *fs_out = ret; error: return err_out; }
// Caller takes ownership of returned object and must delete it when done google::protobuf::io::CodedInputStream* MaprInputCodedBlockFile::CreateCodedStream(uint64 position, uint64 length) { CHECK(is_open_); // Seek to requested position (relative to start of file). CHECK_LT(position, size_); CHECK_LE(position+length, size_); // Starting with MapR V2.1.1, sometimes seek fails and the hdfs connection // must be reset in order to try again. Not sure why this transient error // happens with MapR V2.1.1 but not earlier versions. bool success = false; for (int i=0; i < 10; ++i){ if (hdfsSeek(fs_, file_, position) == 0){ success = true; break; } //LOG(INFO) << "seek attempt failed: " << i; //LOG(INFO) << "path:" << path_ << "\n position: " << position << "\n length: " << length << "\n size: " << size_; // success if returns 0 CHECK_EQ(hdfsCloseFile(fs_, file_), 0); CHECK_EQ(hdfsDisconnect(fs_), 0); std::string host = "default"; fs_ = hdfsConnect(host.c_str(), 0); // use default config file settings CHECK(fs_) << "error connecting to maprfs"; file_ = hdfsOpenFile(fs_, path_.c_str(), O_RDONLY, 0, 0, 0); CHECK(file_ != NULL); sleep(2*i); } CHECK(success); // Create a coded stream (hold it in a scoped ptr to manage deleting). limiting_stream_.reset(NULL); // the destructor references the copying_stream_, so must destroy it before destroying it copying_stream_.reset(new google::protobuf::io::CopyingInputStreamAdaptor(copying_input_stream_.get())); limiting_stream_.reset(new google::protobuf::io::LimitingInputStream(copying_stream_.get(), length)); return new google::protobuf::io::CodedInputStream(limiting_stream_.get()); }
int main(int argc, char **argv) { hdfsFS fs = hdfsConnect("default", 0); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs!\n"); exit(-1); } hdfsFS lfs = hdfsConnect(NULL, 0); if(!lfs) { fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n"); exit(-1); } const char* writePath = "/tmp/testfile.txt"; { //Write tests hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", writePath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", writePath); char* buffer = "Hello, World!"; tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); tOffset currentPos = -1; if ((currentPos = hdfsTell(fs, writeFile)) == -1) { fprintf(stderr, "Failed to get current file position correctly! Got %ld!\n", currentPos); exit(-1); } fprintf(stderr, "Current position: %ld\n", currentPos); if (hdfsFlush(fs, writeFile)) { fprintf(stderr, "Failed to 'flush' %s\n", writePath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", writePath); hdfsCloseFile(fs, writeFile); } { //Read tests const char* readPath = "/tmp/testfile.txt"; int exists = hdfsExists(fs, readPath); if (exists) { fprintf(stderr, "Failed to validate existence of %s\n", readPath); exit(-1); } hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0); if (!readFile) { fprintf(stderr, "Failed to open %s for reading!\n", readPath); exit(-1); } fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile)); tOffset seekPos = 1; if(hdfsSeek(fs, readFile, seekPos)) { fprintf(stderr, "Failed to seek %s for reading!\n", readPath); exit(-1); } tOffset currentPos = -1; if((currentPos = hdfsTell(fs, readFile)) != seekPos) { fprintf(stderr, "Failed to get current file position correctly! Got %ld!\n", currentPos); exit(-1); } fprintf(stderr, "Current position: %ld\n", currentPos); static char buffer[32]; tSize num_read_bytes = hdfsRead(fs, readFile, (void*)buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer, sizeof(buffer)); fprintf(stderr, "Read following %d bytes:\n%s\n", num_read_bytes, buffer); hdfsCloseFile(fs, readFile); } int totalResult = 0; int result = 0; { //Generic file-system operations const char* srcPath = "/tmp/testfile.txt"; const char* dstPath = "/tmp/testfile2.txt"; fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; const char* slashTmp = "/tmp"; const char* newDirectory = "/tmp/newdir"; fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) ? "Failed!" : "Success!")); totalResult += result; char buffer[256]; const char *resp; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!")); totalResult += (resp ? 0 : 1); fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!")); totalResult += (resp ? 0 : 1); fprintf(stderr, "hdfsGetDefaultBlockSize: %ld\n", hdfsGetDefaultBlockSize(fs)); fprintf(stderr, "hdfsGetCapacity: %ld\n", hdfsGetCapacity(fs)); fprintf(stderr, "hdfsGetUsed: %ld\n", hdfsGetUsed(fs)); hdfsFileInfo *fileInfo = NULL; if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) { fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n"); fprintf(stderr, "Name: %s, ", fileInfo->mName); fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind)); fprintf(stderr, "Replication: %d, ", fileInfo->mReplication); fprintf(stderr, "BlockSize: %ld, ", fileInfo->mBlockSize); fprintf(stderr, "Size: %ld, ", fileInfo->mSize); fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod)); fprintf(stderr, "Owner: %s, ", fileInfo->mOwner); fprintf(stderr, "Group: %s, ", fileInfo->mGroup); char permissions[10]; permission_disp(fileInfo->mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions); hdfsFreeFileInfo(fileInfo, 1); } else { totalResult++; fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp); } hdfsFileInfo *fileList = 0; int numEntries = 0; if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) { int i = 0; for(i=0; i < numEntries; ++i) { fprintf(stderr, "Name: %s, ", fileList[i].mName); fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind); fprintf(stderr, "Replication: %d, ", fileList[i].mReplication); fprintf(stderr, "BlockSize: %ld, ", fileList[i].mBlockSize); fprintf(stderr, "Size: %ld, ", fileList[i].mSize); fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod)); fprintf(stderr, "Owner: %s, ", fileList[i].mOwner); fprintf(stderr, "Group: %s, ", fileList[i].mGroup); char permissions[10]; permission_disp(fileList[i].mPermissions, permissions); fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions); } hdfsFreeFileInfo(fileList, numEntries); } else { if (errno) { totalResult++; fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n"); } else { fprintf(stderr, "Empty directory!\n"); } } char*** hosts = hdfsGetHosts(fs, srcPath, 0, 1); if(hosts) { fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n"); int i=0; while(hosts[i]) { int j = 0; while(hosts[i][j]) { fprintf(stderr, "\thosts[%d][%d] - %s\n", i, j, hosts[i][j]); ++j; } ++i; } } else { totalResult++; fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n"); } char *newOwner = "root"; // setting tmp dir to 777 so later when connectAsUser nobody, we can write to it short newPerm = 0666; // chown write fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) ? "Failed!" : "Success!")); totalResult += result; // chmod write fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) ? "Failed!" : "Success!")); totalResult += result; sleep(2); tTime newMtime = time(NULL); tTime newAtime = time(NULL); // utime write fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) ? "Failed!" : "Success!")); totalResult += result; // chown/chmod/utime read hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath); fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!")); totalResult += result; // will later use /tmp/ as a different user so enable it fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr,"newMTime=%ld\n",newMtime); fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod); fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) ? "Failed!" : "Success!")); totalResult += result; // No easy way to turn on access times from hdfs_test right now // fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) ? "Failed!" : "Success!")); // totalResult += result; hdfsFreeFileInfo(finfo, 1); // Clean up fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath)) ? "Failed!" : "Success!")); totalResult += result; fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!")); totalResult += (result ? 0 : 1); } totalResult += (hdfsDisconnect(fs) != 0); { // // Now test as connecting as a specific user // This is only meant to test that we connected as that user, not to test // the actual fs user capabilities. Thus just create a file and read // the owner is correct. const char *tuser = "******"; const char* writePath = "/tmp/usertestfile.txt"; const char **groups = (const char**)malloc(sizeof(char*)* 2); groups[0] = "users"; groups[1] = "nobody"; fs = hdfsConnectAsUser("default", 0, tuser, groups, 2); if(!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser); exit(-1); } hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); if(!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", writePath); exit(-1); } fprintf(stderr, "Opened %s for writing successfully...\n", writePath); char* buffer = "Hello, World!"; tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1); fprintf(stderr, "Wrote %d bytes\n", num_written_bytes); if (hdfsFlush(fs, writeFile)) { fprintf(stderr, "Failed to 'flush' %s\n", writePath); exit(-1); } fprintf(stderr, "Flushed %s successfully!\n", writePath); hdfsCloseFile(fs, writeFile); hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath); fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!")); totalResult += result; } totalResult += (hdfsDisconnect(fs) != 0); if (totalResult != 0) { return -1; } else { return 0; } }
HdfsConnector::HdfsConnector(vector<vector <string> > hdfs_writepath):Connector(hdfs_writepath) { hdfsFS hdfsfs = hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port); fs = hdfsfs; }
int uploadFile(const char *path){ hdfsFS fs = hdfsConnect("default", 0); hdfsFile fd_w; int fd; unsigned long size, i; struct stat fd_s; char *buf_r; const char *filename; /* Get the file size */ if ((fd = open(path, O_RDONLY)) < 0){ perror("Open file failed"); exit(1); } if (fstat(fd, &fd_s) < 0){ perror("Get file stat failed"); exit(1); } size = fd_s.st_size; /* Open the file at hdfs */ filename = strrchr(path, '/'); if (!filename){ filename = path; }else{ filename += 1; } fd_w = hdfsOpenFile(fs, filename, O_WRONLY|O_CREAT, 0, 0, 0); if (!fd_w){ perror("Failed to create file to upload"); exit(1); } /* Write the file */ for (i = 0; i + UPLOAD_BLOCK < size; i += UPLOAD_BLOCK){ buf_r = (char *)mmap(NULL,\ UPLOAD_BLOCK,\ PROT_READ,\ MAP_SHARED,\ fd, i); if (buf_r == MAP_FAILED){ perror("Failed to map memory"); exit(1); } hdfsWrite(fs, fd_w, (void*)buf_r, UPLOAD_BLOCK); if (hdfsFlush(fs, fd_w)){ perror("Failed to flush"); exit(1); } if ((munmap(buf_r, UPLOAD_BLOCK)) < 0){ perror("memory unmap error"); exit(1); } } if (size){ /* To avoid size == 0 */ buf_r = (char *)mmap(NULL,\ size - i,\ PROT_READ,\ MAP_SHARED,\ fd, i); if (buf_r == MAP_FAILED){ perror("Failed to map memory"); exit(1); } hdfsWrite(fs, fd_w, (void*)buf_r, size - i); if (hdfsFlush(fs, fd_w)){ perror("Failed to flush"); exit(1); } if ((munmap(buf_r, size - i)) < 0){ perror("memory unmap error"); exit(1); } } /* Upload end */ close(fd); hdfsCloseFile(fs, fd_w); hdfsDisconnect(fs); return 0; }
/** * Open a connection to the filesystem. The default arguments * should be sufficient for most uses */ hdfs(const std::string& host = "default", tPort port = 0) { filesystem = hdfsConnect(host.c_str(), port); ASSERT_TRUE(filesystem != NULL); } // end of constructor
/* constructor */ static PyObject * pyhdfsFS_open(char* filepath,char mode) { pyhdfsFS *object = PyObject_NEW(pyhdfsFS, &pyhdfsFSType); if (object != NULL) { // Parse HDFS information char* hostport = (char *)malloc(strlen(filepath)+1); char* buf = NULL; char* path; char* portStr; char* host; int hdfsPort; int ret = 0; ret = sscanf(filepath, "hdfs://%s", hostport); host = strtok_r(hostport, ":", &buf); portStr = strtok_r(NULL, "/", &buf); ret = sscanf(portStr, "%d", &hdfsPort); object->fs = hdfsConnect(host, hdfsPort); if (!object->fs) { PyErr_SetString(exception, "Cannot connect to host"); return NULL; } path = (char*) malloc(strlen(buf)+2); path[0] = '/'; memcpy(path+1,buf,strlen(buf)); path[strlen(buf)+1] = '\0'; if (mode == 'r') { if (hdfsExists(object->fs, path) == 0) { object->file = hdfsOpenFile(object->fs, path, O_RDONLY, 0, 0, 0); } else { char* error_message = (char *)calloc(strlen("Cannot open file for read : ") + strlen(path) + 1, sizeof(char)); strcat(error_message, "Cannot open file for read : "); strcat(error_message, path); PyErr_SetString(exception, error_message); free(error_message); return NULL; } } else if (mode == 'w') { if (hdfsExists(object->fs, path) == 0) { char* error_message = (char *)calloc(strlen("Cannot open file for write : ") + strlen(path) + 1, sizeof(char)); strcat(error_message, "Cannot open file for write : "); strcat(error_message, path); PyErr_SetString(exception, error_message); free(error_message); return NULL; } else object->file = hdfsOpenFile(object->fs, path, O_WRONLY, 0, 0, 0); } else { PyErr_SetString(exception, "Invalid mode"); return NULL; } if(!object->file) { char* error_message = (char *)calloc(strlen("Cannot open file : ") + strlen(path) + 1, sizeof(char)); strcat(error_message, "Cannot open file : "); strcat(error_message, path); PyErr_SetString(exception, error_message); free(error_message); return NULL; } free(hostport); free(path); } return (PyObject *)object; }
qioerr hdfs_preadv (void* file, const struct iovec *vector, int count, off_t offset, ssize_t* num_read_out, void* fs) { ssize_t got; ssize_t got_total; qioerr err_out = 0; int i; STARTING_SLOW_SYSCALL; #ifdef HDFS3 const hdfs_file orig_hfl = *to_hdfs_file(file); const hdfs_fs orig_hfs = *to_hdfs_fs(fs); hdfsFS hfs = hdfsConnect(orig_hfs.fs_name, orig_hfs.fs_port); hdfsFile hfl = hdfsOpenFile(hfs, orig_hfl.pathnm, O_RDONLY, 0, 0, 0); //assert connection CREATE_ERROR((hfs == NULL), err_out, ECONNREFUSED, "Unable to read HDFS file", error); if(hfl == NULL) { err_out = qio_mkerror_errno(); goto error; } #endif err_out = 0; got_total = 0; for(i = 0; i < count; i++) { #ifdef HDFS3 hdfsSeek(hfs, hfl, offset+got_total); got = hdfsRead(hfs, hfl, (void*)vector[i].iov_base, vector[i].iov_len); #else got = hdfsPread(to_hdfs_fs(fs)->hfs, to_hdfs_file(file)->file, offset + got_total, (void*)vector[i].iov_base, vector[i].iov_len); #endif if( got != -1 ) { got_total += got; } else { err_out = qio_mkerror_errno(); break; } if(got != (ssize_t)vector[i].iov_len ) { break; } } if( err_out == 0 && got_total == 0 && sys_iov_total_bytes(vector, count) != 0 ) err_out = qio_int_to_err(EEOF); *num_read_out = got_total; #ifdef HDFS3 got = hdfsCloseFile(hfs, hfl); if(got == -1) { err_out = qio_mkerror_errno(); } got = hdfsDisconnect(hfs); if(got == -1) { err_out = qio_mkerror_errno(); } #endif DONE_SLOW_SYSCALL; #ifdef HDFS3 error: #endif return err_out; }
MaprOutputCodedBlockFile::MaprOutputCodedBlockFile() { fs_ = hdfsConnect("default", 0); // use default config file settings CHECK(fs_) << "error connecting to hdfs"; }
int main(int argc, char **argv) { if (argc != 4) { fprintf(stderr, "Usage: hdfs_write <filename> <filesize> <buffersize>\n"); exit(-1); } hdfsFS fs = hdfsConnect("default", 0); if (!fs) { fprintf(stderr, "Oops! Failed to connect to hdfs!\n"); exit(-1); } const char* writeFileName = argv[1]; off_t fileTotalSize = strtoul(argv[2], NULL, 10); long long tmpBufferSize = strtoul(argv[3], NULL, 10); // sanity check if(fileTotalSize == ULONG_MAX && errno == ERANGE) { fprintf(stderr, "invalid file size %s - must be <= %lu\n", argv[2], ULONG_MAX); exit(-3); } // currently libhdfs writes are of tSize which is int32 if(tmpBufferSize > INT_MAX) { fprintf(stderr, "invalid buffer size libhdfs API write chunks must be <= %d\n",INT_MAX); exit(-3); } tSize bufferSize = tmpBufferSize; hdfsFile writeFile = hdfsOpenFile(fs, writeFileName, O_WRONLY, bufferSize, 0, 0); if (!writeFile) { fprintf(stderr, "Failed to open %s for writing!\n", writeFileName); exit(-2); } // data to be written to the file char* buffer = malloc(sizeof(char) * bufferSize); if(buffer == NULL) { fprintf(stderr, "Could not allocate buffer of size %d\n", bufferSize); return -2; } int i = 0; for (i=0; i < bufferSize; ++i) { buffer[i] = 'a' + (i%26); } // write to the file off_t nrRemaining; for (nrRemaining = fileTotalSize; nrRemaining > 0; nrRemaining -= bufferSize ) { tSize curSize = ( bufferSize < nrRemaining ) ? bufferSize : (tSize)nrRemaining; tSize written; if ((written = hdfsWrite(fs, writeFile, (void*)buffer, curSize)) != curSize) { fprintf(stderr, "ERROR: hdfsWrite returned an error on write: %d\n", written); exit(-3); } } free(buffer); hdfsCloseFile(fs, writeFile); hdfsDisconnect(fs); return 0; }
hdfsFS hdfsConnectNewInstance(const char* nn, tPort port) { return hdfsConnect(nn, port); }
/////////////////////////////////////////////////////////////////////////////// // MaprInputCodedBlockFile /////////////////////////////////////////////////////////////////////////////// MaprInputCodedBlockFile::MaprInputCodedBlockFile() { std::string host = "default"; fs_ = hdfsConnect(host.c_str(), 0); // use default config file settings CHECK(fs_) << "error connecting to maprfs"; copying_input_stream_.reset(new MaprCopyingInputStream(this)); }