/** * hdfsGetPathInfo - Get information about a path as a (dynamically * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be * called when the pointer is no longer needed. * * @param fs The configured filesystem handle. * @param path The path of the file. * @return Returns a dynamically-allocated hdfsFileInfo object; * NULL on error. */ hdfsFileInfo* hdfsGetPathInfo(hdfsFS fs, const char* path) { hdfsFileInfo *res = NULL; struct hdfsFS_internal *client = fs; struct hdfs_object *ex = NULL, *fstatus = NULL; char *path_abs = _makeabs(fs, path); fstatus = hdfs_getFileInfo(client->fs_namenode, path_abs, &ex); if (ex) { ERR(EIO, "getFileInfo(): %s", hdfs_exception_get_message(ex)); hdfs_object_free(ex); goto out; } if (fstatus->ob_type == H_NULL) { ERR(ENOENT, "getFileInfo(): %s doesn't exist", path_abs); goto out; } res = malloc(sizeof *res); assert(res); _hadoofus_file_status_to_libhdfs(client->fs_uri, path_abs, fstatus, res); out: if (fstatus) hdfs_object_free(fstatus); if (path_abs != path) free(path_abs); return res; }
/** * hdfsExists - Checks if a given path exists on the filesystem * * @param fs The configured filesystem handle. * @param path The path to look for * @return Returns 0 on success, -1 on error. */ int hdfsExists(hdfsFS fs, const char *path) { struct hdfs_object *s, *ex = NULL; struct hdfsFS_internal *client = fs; int res = 0; char *path_abs = _makeabs(fs, path); s = hdfs_getFileInfo(client->fs_namenode, path_abs, &ex); if (ex) { ERR(EIO, "getFileInfo(): %s", hdfs_exception_get_message(ex)); hdfs_object_free(ex); res = -1; goto out; } if (s->ob_type == H_NULL) { res = -1; errno = ENOENT; } hdfs_object_free(s); out: if (path_abs != path) free(path_abs); return res; }
END_TEST START_TEST(test_getFileInfo) { struct hdfs_object *e = NULL, *fs; fs = hdfs_getFileInfo(h, "/", &e); if (e) ck_abort_msg("exception: %s", hdfs_exception_get_message(e)); ck_assert(!hdfs_object_is_null(fs)); hdfs_object_free(fs); }
END_TEST START_TEST(test_dn_write_file) { const char *tf = "/HADOOFUS_TEST_WRITE_FILE", *client = "HADOOFUS_CLIENT", *err; bool s; struct hdfs_datanode *dn; struct hdfs_object *e = NULL, *bl, *fs, *bls; uint64_t begin, end; s = hdfs_delete(h, tf, false/*recurse*/, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); hdfs_create(h, tf, 0644, client, true/*overwrite*/, false/*createparent*/, 1/*replication*/, blocksz, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); begin = _now(); // write first block (full) bl = hdfs_addBlock(h, tf, client, NULL, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err); ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err); hdfs_object_free(bl); err = hdfs_datanode_write_file(dn, fd, blocksz, 0, _i/*crcs*/); fail_if(err, "error writing block: %s", err); hdfs_datanode_delete(dn); // write second block (partial) bl = hdfs_addBlock(h, tf, client, NULL, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err); ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err); hdfs_object_free(bl); err = hdfs_datanode_write_file(dn, fd, towrite-blocksz, blocksz, _i/*crcs*/); fail_if(err, "error writing block: %s", err); hdfs_datanode_delete(dn); end = _now(); fprintf(stderr, "Wrote %d MB from file in %ld ms%s, %02g MB/s\n", towrite/1024/1024, end - begin, _i? " (with crcs)":"", (double)towrite/(end-begin)/1024*1000/1024); fs = hdfs_getFileInfo(h, tf, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); ck_assert(fs->ob_val._file_status._size == towrite); hdfs_object_free(fs); s = hdfs_complete(h, tf, client, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); ck_assert_msg(s, "did not complete"); bls = hdfs_getBlockLocations(h, tf, 0, towrite, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); begin = _now(); for (int i = 0; i < bls->ob_val._located_blocks._num_blocks; i++) { struct hdfs_object *bl = bls->ob_val._located_blocks._blocks[i]; dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err); ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err); err = hdfs_datanode_read_file(dn, 0/*offset-in-block*/, bl->ob_val._located_block._len, ofd, i*blocksz/*fd offset*/, _i/*crcs*/); hdfs_datanode_delete(dn); if (err == HDFS_DATANODE_ERR_NO_CRCS) { fprintf(stderr, "Warning: test server doesn't support " "CRCs, skipping validation.\n"); _i = 0; // reconnect, try again without validating CRCs (for // isi_hdfs_d) dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err); ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err); err = hdfs_datanode_read_file(dn, 0/*offset-in-block*/, bl->ob_val._located_block._len, ofd, i*blocksz, false/*crcs*/); hdfs_datanode_delete(dn); } fail_if(err, "error reading block: %s", err); } end = _now(); fprintf(stderr, "Read %d MB to file in %ld ms%s, %02g MB/s\n", towrite/1024/1024, end - begin, _i? " (with crcs)":"", (double)towrite/(end-begin)/1024*1000/1024); hdfs_object_free(bls); fail_if(filecmp(fd, ofd, towrite), "read differed from write"); s = hdfs_delete(h, tf, false/*recurse*/, &e); if (e) fail("exception: %s", hdfs_exception_get_message(e)); ck_assert_msg(s, "delete returned false"); }