Exemplo n.º 1
0
bool MaprFileSystem::IsDirectory(const std::string& uri){
  std::string path = GetUriPathOrDie(uri);
  std::string host = "default";
  hdfsFS fs = hdfsConnect(host.c_str(), 0); // use default config file settings
  CHECK(fs);
  hdfsFileInfo* info = hdfsGetPathInfo(fs, path.c_str());
  bool is_directory = false;
  if (info){
    is_directory = (info->mKind == kObjectKindDirectory);
    hdfsFreeFileInfo(info,1);
  }
  else{
    LOG(FATAL) << "uri does not exist: " << uri;
  }
  CHECK_EQ(hdfsDisconnect(fs), 0);
  return is_directory;
}
Exemplo n.º 2
0
void ClaimsHDFS::claimsRead(){
	hdfsFS fs;
	hdfsFile fd;
	string filename="/home/casa/data/kmeans_data.txt";
	fs=hdfsConnect("10.11.1.174",9000);
	fd=hdfsOpenFile(fs,filename.c_str(),O_RDONLY,0,0,0);
	if(!fd){
		cout<<"failed to open hdfs file!!!"<<endl;
	}

	char array[72];
    tSize bytes=hdfsRead(fs,fd,array,72);
	cout<<"string is: "<<array<<endl;

	hdfsCloseFile(fs,fd);
	hdfsDisconnect(fs);
}
Exemplo n.º 3
0
hdfsFS HdfsConnection::getConnection(std::string host, int port)
{
  QString id = _makeId(host, port);
  if (_connections.contains(id))
  {
    return _connections[id];
  }
  else
  {
    hdfsFS fs = hdfsConnect(host.c_str(), port);
    if (fs == NULL)
    {
      throw std::ios_base::failure("Error connecting to HDFS.");
    }
    _connections[id] = fs;
    return fs;
  }
}
Exemplo n.º 4
0
static PyObject *
pyhdfsFS_delete(char* filepath)
{ 
    // Parse HDFS information
    char* hostport = (char *)malloc(strlen(filepath)+1);
    char* buf = NULL;
    char* path;
    char* portStr;  
    char* host;
    int hdfsPort;
    int ret = 0;

    ret = sscanf(filepath, "hdfs://%s", hostport);
    host = strtok_r(hostport, ":", &buf);
    portStr = strtok_r(NULL, "/", &buf);
    ret = sscanf(portStr, "%d", &hdfsPort);

    hdfsFS fs = hdfsConnect(host, hdfsPort);
    if (!fs)
    {
        PyErr_SetString(exception, "Cannot connect to host");
        return exception;
    }

    path = (char*) malloc(strlen(buf)+2);
    path[0] = '/';
    memcpy(path+1,buf,strlen(buf));
    path[strlen(buf)+1] = '\0';

    if (hdfsExists(fs, path) == 0)
    {
        hdfsDelete(fs, path);
    }
    else 
    {
        PyErr_SetString(exception, "Cannot delete file");
        return NULL;
    }

    free(hostport);
    free(path);

    return Py_BuildValue("i",1);
}
Exemplo n.º 5
0
Arquivo: fs.c Projeto: HsuJv/Note
int dfsList(const char* path){
    hdfsFS fs = hdfsConnect("default", 0);
    int i, entries;
    hdfsFileInfo *files, *head;

    /* Get the list info */
    files = hdfsListDirectory(fs, path, &entries);
    if (!files){
        perror("Get directory info");
        exit(-1);
    }
    head = files;

    /* Print the info */
    fprintf(stdout, "%s %-50s %-9s %s\n",
            "Kind", "Name", "Size", "Replicas");

    for (i = 0; i < entries; i++){
        const char* unit[] = {" B", "KB", "MB", "GB", "TB", "PB"};
        double size = files->mSize;
        unsigned int u = 0;

        while (size > 1024){
            u++;
            size /= 1024;
        }

        assert(u < 6);

        fprintf(stdout, "%4c %-50s %-7.2lf%s %8d\n", 
                files->mKind, files->mName,
                size, unit[u],
                files->mReplication);

        files += 1;
    }

    /* List ends */
    hdfsFreeFileInfo(head, entries);
    hdfsDisconnect(fs);
    
    return 0;
}
Exemplo n.º 6
0
int main(int argc, char **argv) {

    if (argc != 4) {
        fprintf(stderr, "Usage: hdfs_read <filename> <filesize> <buffersize>\n");
        exit(-1);
    }
    
    hdfsFS fs = hdfsConnect("default", 0);
    if (!fs) {
        fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
        exit(-1);
    } 
 
    const char* rfile = argv[1];
    tSize fileTotalSize = strtoul(argv[2], NULL, 10);
    tSize bufferSize = strtoul(argv[3], NULL, 10);
   
    hdfsFile readFile = hdfsOpenFile(fs, rfile, O_RDONLY, bufferSize, 0, 0);
    if (!readFile) {
        fprintf(stderr, "Failed to open %s for writing!\n", rfile);
        exit(-2);
    }

    // data to be written to the file
    char* buffer = malloc(sizeof(char) * bufferSize);
    if(buffer == NULL) {
        return -2;
    }
    
    // read from the file
    tSize curSize = bufferSize;
    for (; curSize == bufferSize;) {
        curSize = hdfsRead(fs, readFile, (void*)buffer, curSize);
    }
    

    free(buffer);
    hdfsCloseFile(fs, readFile);
    hdfsDisconnect(fs);

    return 0;
}
Exemplo n.º 7
0
qioerr hdfs_connect(void** fs_out, const char* pathname, int port)
{
  qioerr err_out = 0;
  hdfsFS fs;
  hdfs_fs* ret = (hdfs_fs*)qio_calloc(sizeof(hdfs_fs), 1);

  STARTING_SLOW_SYSCALL;
  fs = hdfsConnect(pathname, port);

  CREATE_ERROR((fs == NULL), err_out, ECONNREFUSED,"Unable to connect HDFS", error);
  DONE_SLOW_SYSCALL;

  ret->hfs = fs;

  DO_INIT_REFCNT(ret);
  ret->fs_name = pathname;
  ret->fs_port = port;
  *fs_out = ret;

error:
  return err_out;
}
Exemplo n.º 8
0
// Caller takes ownership of returned object and must delete it when done
google::protobuf::io::CodedInputStream*
MaprInputCodedBlockFile::CreateCodedStream(uint64 position, uint64 length) {
  CHECK(is_open_);

  // Seek to requested position (relative to start of file).
  CHECK_LT(position, size_);
  CHECK_LE(position+length, size_);

  // Starting with MapR V2.1.1, sometimes seek fails and the hdfs connection
  // must be reset in order to try again.  Not sure why this transient error
  // happens with MapR V2.1.1 but not earlier versions.
  bool success = false;
  for (int i=0; i < 10; ++i){
   if (hdfsSeek(fs_, file_, position) == 0){
     success = true;
     break;
   }
    //LOG(INFO) << "seek attempt failed: " << i;
    //LOG(INFO) << "path:" << path_ << "\n position: " << position << "\n length: " << length << "\n size: " << size_;  // success if returns 0
    CHECK_EQ(hdfsCloseFile(fs_, file_), 0);
    CHECK_EQ(hdfsDisconnect(fs_), 0);
    std::string host = "default";
    fs_ = hdfsConnect(host.c_str(), 0); // use default config file settings
    CHECK(fs_) << "error connecting to maprfs";
    file_ = hdfsOpenFile(fs_, path_.c_str(), O_RDONLY, 0, 0, 0);
    CHECK(file_ != NULL);
    sleep(2*i);
  }
  CHECK(success);

  // Create a coded stream (hold it in a scoped ptr to manage deleting).
  limiting_stream_.reset(NULL); // the destructor references the copying_stream_, so must destroy it before destroying it
  copying_stream_.reset(new google::protobuf::io::CopyingInputStreamAdaptor(copying_input_stream_.get()));
  limiting_stream_.reset(new google::protobuf::io::LimitingInputStream(copying_stream_.get(), length));
  return new google::protobuf::io::CodedInputStream(limiting_stream_.get());
}
Exemplo n.º 9
0
int main(int argc, char **argv) {

    hdfsFS fs = hdfsConnect("default", 0);
    if(!fs) {
        fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
        exit(-1);
    } 
 
    hdfsFS lfs = hdfsConnect(NULL, 0);
    if(!lfs) {
        fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
        exit(-1);
    } 
 
        const char* writePath = "/tmp/testfile.txt";
    {
        //Write tests
        
        
        hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
        if(!writeFile) {
            fprintf(stderr, "Failed to open %s for writing!\n", writePath);
            exit(-1);
        }
        fprintf(stderr, "Opened %s for writing successfully...\n", writePath);

        char* buffer = "Hello, World!";
        tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
        fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);

        tOffset currentPos = -1;
        if ((currentPos = hdfsTell(fs, writeFile)) == -1) {
            fprintf(stderr, 
                    "Failed to get current file position correctly! Got %ld!\n",
                    currentPos);
            exit(-1);
        }
        fprintf(stderr, "Current position: %ld\n", currentPos);

        if (hdfsFlush(fs, writeFile)) {
            fprintf(stderr, "Failed to 'flush' %s\n", writePath); 
            exit(-1);
        }
        fprintf(stderr, "Flushed %s successfully!\n", writePath); 

        hdfsCloseFile(fs, writeFile);
    }

    {
        //Read tests
        
        const char* readPath = "/tmp/testfile.txt";
        int exists = hdfsExists(fs, readPath);

        if (exists) {
          fprintf(stderr, "Failed to validate existence of %s\n", readPath);
          exit(-1);
        }

        hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
        if (!readFile) {
            fprintf(stderr, "Failed to open %s for reading!\n", readPath);
            exit(-1);
        }

        fprintf(stderr, "hdfsAvailable: %d\n", hdfsAvailable(fs, readFile));

        tOffset seekPos = 1;
        if(hdfsSeek(fs, readFile, seekPos)) {
            fprintf(stderr, "Failed to seek %s for reading!\n", readPath);
            exit(-1);
        }

        tOffset currentPos = -1;
        if((currentPos = hdfsTell(fs, readFile)) != seekPos) {
            fprintf(stderr, 
                    "Failed to get current file position correctly! Got %ld!\n", 
                    currentPos);
            exit(-1);
        }
        fprintf(stderr, "Current position: %ld\n", currentPos);

        static char buffer[32];
        tSize num_read_bytes = hdfsRead(fs, readFile, (void*)buffer, 
                sizeof(buffer));
        fprintf(stderr, "Read following %d bytes:\n%s\n", 
                num_read_bytes, buffer);

        num_read_bytes = hdfsPread(fs, readFile, 0, (void*)buffer, 
                sizeof(buffer));
        fprintf(stderr, "Read following %d bytes:\n%s\n", 
                num_read_bytes, buffer);

        hdfsCloseFile(fs, readFile);
    }

    int totalResult = 0;
    int result = 0;
    {
        //Generic file-system operations

        const char* srcPath = "/tmp/testfile.txt";
        const char* dstPath = "/tmp/testfile2.txt";

        fprintf(stderr, "hdfsCopy(remote-local): %s\n", ((result = hdfsCopy(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsMove(local-local): %s\n", ((result = hdfsMove(lfs, srcPath, lfs, dstPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsMove(remote-local): %s\n", ((result = hdfsMove(fs, srcPath, lfs, srcPath)) ? "Failed!" : "Success!"));
        totalResult += result;

        fprintf(stderr, "hdfsRename: %s\n", ((result = hdfsRename(fs, dstPath, srcPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsCopy(remote-remote): %s\n", ((result = hdfsCopy(fs, srcPath, fs, dstPath)) ? "Failed!" : "Success!"));
        totalResult += result;

        const char* slashTmp = "/tmp";
        const char* newDirectory = "/tmp/newdir";
        fprintf(stderr, "hdfsCreateDirectory: %s\n", ((result = hdfsCreateDirectory(fs, newDirectory)) ? "Failed!" : "Success!"));
        totalResult += result;

        fprintf(stderr, "hdfsSetReplication: %s\n", ((result = hdfsSetReplication(fs, srcPath, 2)) ? "Failed!" : "Success!"));
        totalResult += result;

        char buffer[256];
        const char *resp;
        fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
        totalResult += (resp ? 0 : 1);
        fprintf(stderr, "hdfsSetWorkingDirectory: %s\n", ((result = hdfsSetWorkingDirectory(fs, slashTmp)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsGetWorkingDirectory: %s\n", ((resp = hdfsGetWorkingDirectory(fs, buffer, sizeof(buffer))) ? buffer : "Failed!"));
        totalResult += (resp ? 0 : 1);

        fprintf(stderr, "hdfsGetDefaultBlockSize: %ld\n", hdfsGetDefaultBlockSize(fs));
        fprintf(stderr, "hdfsGetCapacity: %ld\n", hdfsGetCapacity(fs));
        fprintf(stderr, "hdfsGetUsed: %ld\n", hdfsGetUsed(fs));

        hdfsFileInfo *fileInfo = NULL;
        if((fileInfo = hdfsGetPathInfo(fs, slashTmp)) != NULL) {
            fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
            fprintf(stderr, "Name: %s, ", fileInfo->mName);
            fprintf(stderr, "Type: %c, ", (char)(fileInfo->mKind));
            fprintf(stderr, "Replication: %d, ", fileInfo->mReplication);
            fprintf(stderr, "BlockSize: %ld, ", fileInfo->mBlockSize);
            fprintf(stderr, "Size: %ld, ", fileInfo->mSize);
            fprintf(stderr, "LastMod: %s", ctime(&fileInfo->mLastMod)); 
            fprintf(stderr, "Owner: %s, ", fileInfo->mOwner);
            fprintf(stderr, "Group: %s, ", fileInfo->mGroup);
            char permissions[10];
            permission_disp(fileInfo->mPermissions, permissions);
            fprintf(stderr, "Permissions: %d (%s)\n", fileInfo->mPermissions, permissions);
            hdfsFreeFileInfo(fileInfo, 1);
        } else {
            totalResult++;
            fprintf(stderr, "waah! hdfsGetPathInfo for %s - FAILED!\n", slashTmp);
        }

        hdfsFileInfo *fileList = 0;
        int numEntries = 0;
        if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
            int i = 0;
            for(i=0; i < numEntries; ++i) {
                fprintf(stderr, "Name: %s, ", fileList[i].mName);
                fprintf(stderr, "Type: %c, ", (char)fileList[i].mKind);
                fprintf(stderr, "Replication: %d, ", fileList[i].mReplication);
                fprintf(stderr, "BlockSize: %ld, ", fileList[i].mBlockSize);
                fprintf(stderr, "Size: %ld, ", fileList[i].mSize);
                fprintf(stderr, "LastMod: %s", ctime(&fileList[i].mLastMod));
                fprintf(stderr, "Owner: %s, ", fileList[i].mOwner);
                fprintf(stderr, "Group: %s, ", fileList[i].mGroup);
                char permissions[10];
                permission_disp(fileList[i].mPermissions, permissions);
                fprintf(stderr, "Permissions: %d (%s)\n", fileList[i].mPermissions, permissions);
            }
            hdfsFreeFileInfo(fileList, numEntries);
        } else {
            if (errno) {
                totalResult++;
                fprintf(stderr, "waah! hdfsListDirectory - FAILED!\n");
            } else {
                fprintf(stderr, "Empty directory!\n");
            }
        }

        char*** hosts = hdfsGetHosts(fs, srcPath, 0, 1);
        if(hosts) {
            fprintf(stderr, "hdfsGetHosts - SUCCESS! ... \n");
            int i=0; 
            while(hosts[i]) {
                int j = 0;
                while(hosts[i][j]) {
                    fprintf(stderr, 
                            "\thosts[%d][%d] - %s\n", i, j, hosts[i][j]);
                    ++j;
                }
                ++i;
            }
        } else {
            totalResult++;
            fprintf(stderr, "waah! hdfsGetHosts - FAILED!\n");
        }
       
        char *newOwner = "root";
        // setting tmp dir to 777 so later when connectAsUser nobody, we can write to it
        short newPerm = 0666;

        // chown write
        fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, NULL, "users")) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsChown: %s\n", ((result = hdfsChown(fs, writePath, newOwner, NULL)) ? "Failed!" : "Success!"));
        totalResult += result;
        // chmod write
        fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, writePath, newPerm)) ? "Failed!" : "Success!"));
        totalResult += result;



        sleep(2);
        tTime newMtime = time(NULL);
        tTime newAtime = time(NULL);

        // utime write
        fprintf(stderr, "hdfsUtime: %s\n", ((result = hdfsUtime(fs, writePath, newMtime, newAtime)) ? "Failed!" : "Success!"));

        totalResult += result;

        // chown/chmod/utime read
        hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);

        fprintf(stderr, "hdfsChown read: %s\n", ((result = (strcmp(finfo->mOwner, newOwner) != 0)) ? "Failed!" : "Success!"));
        totalResult += result;

        fprintf(stderr, "hdfsChmod read: %s\n", ((result = (finfo->mPermissions != newPerm)) ? "Failed!" : "Success!"));
        totalResult += result;

        // will later use /tmp/ as a different user so enable it
        fprintf(stderr, "hdfsChmod: %s\n", ((result = hdfsChmod(fs, "/tmp/", 0777)) ? "Failed!" : "Success!"));
        totalResult += result;

        fprintf(stderr,"newMTime=%ld\n",newMtime);
        fprintf(stderr,"curMTime=%ld\n",finfo->mLastMod);


        fprintf(stderr, "hdfsUtime read (mtime): %s\n", ((result = (finfo->mLastMod != newMtime)) ? "Failed!" : "Success!"));
        totalResult += result;

        // No easy way to turn on access times from hdfs_test right now
        //        fprintf(stderr, "hdfsUtime read (atime): %s\n", ((result = (finfo->mLastAccess != newAtime)) ? "Failed!" : "Success!"));
        //        totalResult += result;

        hdfsFreeFileInfo(finfo, 1);

        // Clean up
        fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, newDirectory)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(fs, srcPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, srcPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsDelete: %s\n", ((result = hdfsDelete(lfs, dstPath)) ? "Failed!" : "Success!"));
        totalResult += result;
        fprintf(stderr, "hdfsExists: %s\n", ((result = hdfsExists(fs, newDirectory)) ? "Success!" : "Failed!"));
        totalResult += (result ? 0 : 1);
    }


    totalResult += (hdfsDisconnect(fs) != 0);

    {
      //
      // Now test as connecting as a specific user
      // This is only meant to test that we connected as that user, not to test
      // the actual fs user capabilities. Thus just create a file and read
      // the owner is correct.

      const char *tuser = "******";
      const char* writePath = "/tmp/usertestfile.txt";
      const char **groups =  (const char**)malloc(sizeof(char*)* 2);
      groups[0] = "users";
      groups[1] = "nobody";

      fs = hdfsConnectAsUser("default", 0, tuser, groups, 2);
      if(!fs) {
        fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
        exit(-1);
      } 

        hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
        if(!writeFile) {
            fprintf(stderr, "Failed to open %s for writing!\n", writePath);
            exit(-1);
        }
        fprintf(stderr, "Opened %s for writing successfully...\n", writePath);

        char* buffer = "Hello, World!";
        tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
        fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);

        if (hdfsFlush(fs, writeFile)) {
            fprintf(stderr, "Failed to 'flush' %s\n", writePath); 
            exit(-1);
        }
        fprintf(stderr, "Flushed %s successfully!\n", writePath); 

        hdfsCloseFile(fs, writeFile);

        hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
        fprintf(stderr, "hdfs new file user is correct: %s\n", ((result = (strcmp(finfo->mOwner, tuser) != 0)) ? "Failed!" : "Success!"));
        totalResult += result;
    }
    
    totalResult += (hdfsDisconnect(fs) != 0);

    if (totalResult != 0) {
        return -1;
    } else {
        return 0;
    }
}
Exemplo n.º 10
0
HdfsConnector::HdfsConnector(vector<vector <string> > hdfs_writepath):Connector(hdfs_writepath) {
	hdfsFS hdfsfs = hdfsConnect(Config::hdfs_master_ip.c_str(),Config::hdfs_master_port);
	fs = hdfsfs;

}
Exemplo n.º 11
0
Arquivo: upload.c Projeto: HsuJv/Note
int uploadFile(const char *path){
    hdfsFS fs = hdfsConnect("default", 0);
    hdfsFile fd_w;
    int fd;
    unsigned long size, i;
    struct stat fd_s;
    char *buf_r;
    const char *filename;

    /* Get the file size */
    if ((fd = open(path, O_RDONLY)) < 0){
        perror("Open file failed");
        exit(1);
    }

    if (fstat(fd, &fd_s) < 0){
        perror("Get file stat failed");
        exit(1);
    }
    
    size = fd_s.st_size;

    /* Open the file at hdfs */
    filename = strrchr(path, '/');
    if (!filename){
        filename = path;
    }else{
        filename += 1;
    }
    
    fd_w = hdfsOpenFile(fs, filename, O_WRONLY|O_CREAT, 0, 0, 0);
    if (!fd_w){
        perror("Failed to create file to upload");
        exit(1);
    }    

    /* Write the file */
    for (i = 0; i + UPLOAD_BLOCK < size; i += UPLOAD_BLOCK){
        buf_r = (char *)mmap(NULL,\
                UPLOAD_BLOCK,\
                PROT_READ,\
                MAP_SHARED,\
                fd, i);
        if (buf_r == MAP_FAILED){
            perror("Failed to map memory");
            exit(1);
        }

        hdfsWrite(fs, fd_w, (void*)buf_r, UPLOAD_BLOCK);
        if (hdfsFlush(fs, fd_w)){
            perror("Failed to flush");
            exit(1);
        }
        
        if ((munmap(buf_r, UPLOAD_BLOCK)) < 0){
            perror("memory unmap error");
            exit(1);
        }
    }
    if (size){
        /* To avoid size == 0 */
        buf_r = (char *)mmap(NULL,\
                    size - i,\
                    PROT_READ,\
                    MAP_SHARED,\
                    fd, i);
        if (buf_r == MAP_FAILED){
            perror("Failed to map memory");
            exit(1);
        }
    
        hdfsWrite(fs, fd_w, (void*)buf_r, size - i);
        if (hdfsFlush(fs, fd_w)){
            perror("Failed to flush");
            exit(1);
        }
            
        if ((munmap(buf_r, size - i)) < 0){
            perror("memory unmap error");
            exit(1);
        }
    }
    
    /* Upload end */
    close(fd);
    hdfsCloseFile(fs, fd_w);
    hdfsDisconnect(fs);
    return 0;
}
Exemplo n.º 12
0
 /**
  * Open a connection to the filesystem. The default arguments
  * should be sufficient for most uses 
  */
 hdfs(const std::string& host = "default", tPort port = 0) {
   filesystem =  hdfsConnect(host.c_str(), port);
   ASSERT_TRUE(filesystem != NULL); 
 } // end of constructor
Exemplo n.º 13
0
/* constructor */
static PyObject *
pyhdfsFS_open(char* filepath,char mode)
{ 
    pyhdfsFS *object = PyObject_NEW(pyhdfsFS, &pyhdfsFSType);

    if (object != NULL)
    {
        // Parse HDFS information
        char* hostport = (char *)malloc(strlen(filepath)+1);		
        char* buf = NULL;
        char* path;
        char* portStr;  
        char* host;
        int hdfsPort;
        int ret = 0;

        ret = sscanf(filepath, "hdfs://%s", hostport);
        host = strtok_r(hostport, ":", &buf);
        portStr = strtok_r(NULL, "/", &buf);
        ret = sscanf(portStr, "%d", &hdfsPort);

        object->fs = hdfsConnect(host, hdfsPort);

        if (!object->fs)
        {
            PyErr_SetString(exception, "Cannot connect to host");
            return NULL;
        }

        path = (char*) malloc(strlen(buf)+2);
        path[0] = '/';
        memcpy(path+1,buf,strlen(buf));
        path[strlen(buf)+1] = '\0';

        if (mode == 'r')
        {
            if (hdfsExists(object->fs, path) == 0)
            {
                object->file = hdfsOpenFile(object->fs, path, O_RDONLY, 0, 0, 0);    
            }
            else 
            {
                char* error_message = (char *)calloc(strlen("Cannot open file for read : ") + strlen(path) + 1, 
                        sizeof(char));
                strcat(error_message, "Cannot open file for read : ");
                strcat(error_message, path);

                PyErr_SetString(exception, error_message);
                free(error_message);
                return NULL;
            }
        }
        else if (mode == 'w')
        {
            if (hdfsExists(object->fs, path) == 0)
            {            
                char* error_message = (char *)calloc(strlen("Cannot open file for write : ") + strlen(path) + 1, 
                        sizeof(char));
                strcat(error_message, "Cannot open file for write : ");
                strcat(error_message, path);

                PyErr_SetString(exception, error_message);
                free(error_message);
                return NULL;
            }             
            else 
               object->file = hdfsOpenFile(object->fs, path, O_WRONLY, 0, 0, 0);    
        }
        else
        {
            PyErr_SetString(exception, "Invalid mode");
            return NULL;
        }

        if(!object->file) 
        {
            char* error_message = (char *)calloc(strlen("Cannot open file : ") + strlen(path) + 1, 
                        sizeof(char));
            strcat(error_message, "Cannot open file : ");
            strcat(error_message, path);

            PyErr_SetString(exception, error_message);
            free(error_message);

            return NULL;
        }        
        free(hostport);
		free(path);
    }
    return (PyObject *)object;
}
Exemplo n.º 14
0
qioerr hdfs_preadv (void* file, const struct iovec *vector, int count, off_t offset, ssize_t* num_read_out, void* fs)
{
  ssize_t got;
  ssize_t got_total;
  qioerr err_out = 0;
  int i;

  STARTING_SLOW_SYSCALL;

#ifdef HDFS3

  const hdfs_file orig_hfl = *to_hdfs_file(file);
  const hdfs_fs orig_hfs = *to_hdfs_fs(fs);

  hdfsFS hfs = hdfsConnect(orig_hfs.fs_name, orig_hfs.fs_port);
  hdfsFile hfl = hdfsOpenFile(hfs, orig_hfl.pathnm, O_RDONLY, 0, 0, 0);

  //assert connection
  CREATE_ERROR((hfs == NULL), err_out, ECONNREFUSED, "Unable to read HDFS file", error);

  if(hfl == NULL) {
    err_out = qio_mkerror_errno();
    goto error;
  }

#endif

  err_out = 0;
  got_total = 0;
  for(i = 0; i < count; i++) {

#ifdef HDFS3
    hdfsSeek(hfs, hfl, offset+got_total);
    got = hdfsRead(hfs, hfl, (void*)vector[i].iov_base, vector[i].iov_len);
#else
    got = hdfsPread(to_hdfs_fs(fs)->hfs, to_hdfs_file(file)->file, offset + got_total, (void*)vector[i].iov_base, vector[i].iov_len);
#endif

    if( got != -1 ) {
      got_total += got;
    } else {
      err_out = qio_mkerror_errno();
      break;
    }
    if(got != (ssize_t)vector[i].iov_len ) {
      break;
    }
  }

  if( err_out == 0 && got_total == 0 && sys_iov_total_bytes(vector, count) != 0 )
    err_out = qio_int_to_err(EEOF);

  *num_read_out = got_total;

#ifdef HDFS3
  got = hdfsCloseFile(hfs, hfl);
  if(got == -1) { err_out = qio_mkerror_errno(); }

  got = hdfsDisconnect(hfs);
  if(got == -1) { err_out = qio_mkerror_errno(); }

#endif

  DONE_SLOW_SYSCALL;

#ifdef HDFS3
error:
#endif
  return err_out;
}
Exemplo n.º 15
0
MaprOutputCodedBlockFile::MaprOutputCodedBlockFile() {
  fs_ = hdfsConnect("default", 0); // use default config file settings
  CHECK(fs_) << "error connecting to hdfs";
}
Exemplo n.º 16
0
int main(int argc, char **argv) {

    if (argc != 4) {
        fprintf(stderr, "Usage: hdfs_write <filename> <filesize> <buffersize>\n");
        exit(-1);
    }
    
    hdfsFS fs = hdfsConnect("default", 0);
    if (!fs) {
        fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
        exit(-1);
    } 
 
    const char* writeFileName = argv[1];
    off_t fileTotalSize = strtoul(argv[2], NULL, 10);
    long long tmpBufferSize = strtoul(argv[3], NULL, 10);

    // sanity check
    if(fileTotalSize == ULONG_MAX && errno == ERANGE) {
      fprintf(stderr, "invalid file size %s - must be <= %lu\n", argv[2], ULONG_MAX);
      exit(-3);
    }

    // currently libhdfs writes are of tSize which is int32
    if(tmpBufferSize > INT_MAX) {
      fprintf(stderr, "invalid buffer size libhdfs API write chunks must be <= %d\n",INT_MAX);
      exit(-3);
    }

    tSize bufferSize = tmpBufferSize;

    hdfsFile writeFile = hdfsOpenFile(fs, writeFileName, O_WRONLY, bufferSize, 0, 0);
    if (!writeFile) {
        fprintf(stderr, "Failed to open %s for writing!\n", writeFileName);
        exit(-2);
    }

    // data to be written to the file
    char* buffer = malloc(sizeof(char) * bufferSize);
    if(buffer == NULL) {
        fprintf(stderr, "Could not allocate buffer of size %d\n", bufferSize);
        return -2;
    }
    int i = 0;
    for (i=0; i < bufferSize; ++i) {
        buffer[i] = 'a' + (i%26);
    }

    // write to the file
    off_t nrRemaining;
    for (nrRemaining = fileTotalSize; nrRemaining > 0; nrRemaining -= bufferSize ) {
      tSize curSize = ( bufferSize < nrRemaining ) ? bufferSize : (tSize)nrRemaining; 
      tSize written;
      if ((written = hdfsWrite(fs, writeFile, (void*)buffer, curSize)) != curSize) {
        fprintf(stderr, "ERROR: hdfsWrite returned an error on write: %d\n", written);
        exit(-3);
      }
    }

    free(buffer);
    hdfsCloseFile(fs, writeFile);
    hdfsDisconnect(fs);

    return 0;
}
Exemplo n.º 17
0
hdfsFS hdfsConnectNewInstance(const char* nn, tPort port)
{
    return hdfsConnect(nn, port);
}
Exemplo n.º 18
0
///////////////////////////////////////////////////////////////////////////////
// MaprInputCodedBlockFile
///////////////////////////////////////////////////////////////////////////////
MaprInputCodedBlockFile::MaprInputCodedBlockFile() {
  std::string host = "default";
  fs_ = hdfsConnect(host.c_str(), 0); // use default config file settings
  CHECK(fs_) << "error connecting to maprfs";
  copying_input_stream_.reset(new MaprCopyingInputStream(this));
}