END_TEST
START_TEST(test_setReplication)
{
bool s;
struct hdfs_object *e = NULL;
const char *tf = "/HADOOFUS_TEST_SETREPLICATION",
*client = "HADOOFUS_CLIENT";
// Create the file first
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
s = hdfs_setReplication(h, tf, 2, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "setReplication returned false");
// Cleanup
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_recoverLease)
{
bool s;
struct hdfs_object *e = NULL;
const char *tf = "/HADOOFUS_TEST_RECOVERLEASE",
*client = "HADOOFUS_CLIENT",
*client2 = "HADOOFUS_CLIENT_2";
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
s = hdfs_recoverLease(h, tf, client2, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(!s, "recoverLease returned true");
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_append)
{
bool s;
struct hdfs_object *e = NULL, *lb;
const char *tf = "/HADOOFUS_TEST_APPEND",
*client = "HADOOFUS_CLIENT";
// Create the file first
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
s = hdfs_complete(h, tf, client, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "complete returned false");
// Open for appending
lb = hdfs_append(h, tf, client, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
hdfs_object_free(lb);
// Cleanup
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_getContentSummary)
{
bool s;
struct hdfs_object *e = NULL, *cs;
const char *tf = "/HADOOFUS_TEST_CSDIR";
s = hdfs_mkdirs(h, tf, 0755, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "mkdirs returned false");
cs = hdfs_getContentSummary(h, tf, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert(!hdfs_object_is_null(cs));
hdfs_object_free(cs);
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_admin_functions2)
{
struct hdfs_object *e, *e2;
e = e2 = NULL;
hdfs_metaSave(h, "/HADOOFUS_TEST_METASAVE", &e);
(void)hdfs_delete(h, "/HADOOFUS_TEST_METASAVE", false, &e2);
if (e2)
hdfs_object_free(e2);
if (e)
#if 0
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
#else
hdfs_object_free(e);
#endif
(void)hdfs_isFileClosed(h, "/BOGUS", &e);
if (e)
hdfs_object_free(e);
hdfs_setBalancerBandwidth(h, 100000000, &e);
if (e)
#if 0
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
#else
hdfs_object_free(e);
#endif
}
END_TEST
START_TEST(test_addBlock)
{
bool s;
struct hdfs_object *e = NULL, *lb;
const char *tf = "/HADOOFUS_TEST_ADDBLOCK",
*client = "HADOOFUS_CLIENT";
// Create the file first
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
mark_point();
lb = hdfs_addBlock(h, tf, client, NULL, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert(!hdfs_object_is_null(lb));
hdfs_object_free(lb);
mark_point();
// Cleanup
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_symlinks)
{
const char *tl = "/HADOOFUS_TEST_SYMLINKS",
*td = "/HADOOFUS_TEST_BOGUS";
struct hdfs_object *targ, *e, *e2, *fs;
e = e2 = NULL;
hdfs2_createSymlink(h, td, tl, 0755, false, &e);
if (e)
goto err;
targ = hdfs2_getLinkTarget(h, tl, &e);
if (e)
goto err;
hdfs_object_free(targ);
fs = hdfs2_getFileLinkInfo(h, tl, &e);
if (e)
goto err;
hdfs_object_free(fs);
err:
hdfs_delete(h, tl, false, &e2);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
if (e2)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
}
END_TEST
START_TEST(test_complete)
{
bool s;
struct hdfs_object *e = NULL;
const char *tf = "/HADOOFUS_TEST_COMPLETE",
*client = "HADOOFUS_CLIENT";
// Create the file first
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
// XXX this must be updated to cover v2.0+ (last_block/fileid)
s = hdfs_complete(h, tf, client, NULL, 0, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "complete returned false");
// Cleanup
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
/**
* hdfsMove - Move file from one filesystem to another.
*
* @param srcFS The handle to source filesystem.
* @param src The path of source file.
* @param dstFS The handle to destination filesystem.
* @param dst The path of destination file.
* @return Returns 0 on success, -1 on error.
*/
int
hdfsMove(hdfsFS srcFS_, const char* src, hdfsFS dstFS, const char* dst)
{
int res = -1;
bool b;
struct hdfs_object *ex = NULL;
struct hdfsFS_internal *srcFS = srcFS_;
char *src_abs, *dst_abs;
src_abs = _makeabs(srcFS, src);
dst_abs = _makeabs(dstFS, dst);
// Yeah this comparison isn't perfect. We don't have anything better.
if (srcFS_ == dstFS) {
b = hdfs_rename(srcFS->fs_namenode, src_abs, dst_abs, &ex);
if (ex) {
ERR(EIO, "rename failed: %s", hdfs_exception_get_message(ex));
goto out;
}
if (!b)
WARN("rename of '%s' returned false", src_abs);
} else {
res = hdfsCopy(srcFS_, src_abs, dstFS, dst_abs);
if (res == -1) {
ERR(errno, "hdfsCopy failed");
goto out;
}
b = hdfs_delete(srcFS->fs_namenode, src_abs, false/*recurse*/, &ex);
if (ex) {
ERR(EIO, "delete failed: %s", hdfs_exception_get_message(ex));
goto out;
}
if (!b)
WARN("delete of '%s' returned false", src_abs);
}
res = 0;
out:
if (src_abs != src)
free(src_abs);
if (dst_abs != dst)
free(dst_abs);
if (ex)
hdfs_object_free(ex);
return res;
}
END_TEST
START_TEST(test_mkdirs)
{
bool s;
struct hdfs_object *e = NULL;
const char *tf = "/HADOOFUS_TEST_MKDIRS";
s = hdfs_mkdirs(h, tf, 0755, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "mkdirs returned false");
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_abandonBlock)
{
bool s;
struct hdfs_object *e = NULL, *lb, *bl;
const char *tf = "/HADOOFUS_TEST_ABANDONBLOCK",
*client = "HADOOFUS_CLIENT";
// Create the file first
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
mark_point();
// XXX this must be updated to cover v2.0+ (last_block/fileid)
lb = hdfs_addBlock(h, tf, client, NULL, NULL, 0, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert(!hdfs_object_is_null(lb));
mark_point();
bl = hdfs_block_from_located_block(lb);
hdfs_object_free(lb);
mark_point();
hdfs_abandonBlock(h, bl, tf, client, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
hdfs_object_free(bl);
mark_point();
// Cleanup
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
/**
* hdfsDelete - Delete file.
*
* @param fs The configured filesystem handle.
* @param path The path of the file.
* @return Returns 0 on success, -1 on error.
*/
int
hdfsDelete(hdfsFS fs, const char* path)
{
struct hdfs_object *ex = NULL;
struct hdfsFS_internal *client = fs;
char *path_abs = _makeabs(fs, path);
int res = 0;
/*bool b = */hdfs_delete(client->fs_namenode, path_abs, true/*recurse*/, &ex);
if (ex) {
ERR(EIO, "delete(): %s", hdfs_exception_get_message(ex));
hdfs_object_free(ex);
res = -1;
}
if (path_abs != path)
free(path_abs);
return res;
}
END_TEST
START_TEST(test_getBlockLocations2)
{
struct hdfs_object *e = NULL, *e2 = NULL, *bls;
const char *tf = "/HADOOFUS_TEST_GET_BLOCK_LOCATIONS2",
*client = "HADOOFUS_CLIENT";
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
bls = hdfs_getBlockLocations(h, tf, 0L, 1000L, &e);
hdfs_delete(h, tf, false/*recurse*/, &e2);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
if (e2)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e2));
ck_assert_msg(bls->ob_type == H_LOCATED_BLOCKS);
}
END_TEST
START_TEST(test_setTimes)
{
bool s;
struct hdfs_object *e = NULL;
const char *tf = "/HADOOFUS_TEST_SETTIMES",
*client = "HADOOFUS_CLIENT";
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
hdfs_setTimes(h, tf, -1, -1, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_getPreferredBlockSize)
{
bool s;
struct hdfs_object *e = NULL;
const char *tf = "/HADOOFUS_TEST_GETPREFERREDBLOCKSIZE",
*client = "HADOOFUS_CLIENT";
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, 64*1024*1024, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
/*bs = */hdfs_getPreferredBlockSize(h, tf, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
ck_abort_msg("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
END_TEST
START_TEST(test_dn_write_file)
{
const char *tf = "/HADOOFUS_TEST_WRITE_FILE",
*client = "HADOOFUS_CLIENT", *err;
bool s;
struct hdfs_datanode *dn;
struct hdfs_object *e = NULL, *bl, *fs, *bls;
uint64_t begin, end;
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
hdfs_create(h, tf, 0644, client, true/*overwrite*/,
false/*createparent*/, 1/*replication*/, blocksz, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
begin = _now();
// write first block (full)
bl = hdfs_addBlock(h, tf, client, NULL, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err);
ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err);
hdfs_object_free(bl);
err = hdfs_datanode_write_file(dn, fd, blocksz, 0, _i/*crcs*/);
fail_if(err, "error writing block: %s", err);
hdfs_datanode_delete(dn);
// write second block (partial)
bl = hdfs_addBlock(h, tf, client, NULL, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err);
ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err);
hdfs_object_free(bl);
err = hdfs_datanode_write_file(dn, fd, towrite-blocksz, blocksz, _i/*crcs*/);
fail_if(err, "error writing block: %s", err);
hdfs_datanode_delete(dn);
end = _now();
fprintf(stderr, "Wrote %d MB from file in %ld ms%s, %02g MB/s\n",
towrite/1024/1024, end - begin, _i? " (with crcs)":"",
(double)towrite/(end-begin)/1024*1000/1024);
fs = hdfs_getFileInfo(h, tf, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
ck_assert(fs->ob_val._file_status._size == towrite);
hdfs_object_free(fs);
s = hdfs_complete(h, tf, client, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "did not complete");
bls = hdfs_getBlockLocations(h, tf, 0, towrite, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
begin = _now();
for (int i = 0; i < bls->ob_val._located_blocks._num_blocks; i++) {
struct hdfs_object *bl =
bls->ob_val._located_blocks._blocks[i];
dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err);
ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err);
err = hdfs_datanode_read_file(dn, 0/*offset-in-block*/,
bl->ob_val._located_block._len,
ofd,
i*blocksz/*fd offset*/,
_i/*crcs*/);
hdfs_datanode_delete(dn);
if (err == HDFS_DATANODE_ERR_NO_CRCS) {
fprintf(stderr, "Warning: test server doesn't support "
"CRCs, skipping validation.\n");
_i = 0;
// reconnect, try again without validating CRCs (for
// isi_hdfs_d)
dn = hdfs_datanode_new(bl, client, HDFS_DATANODE_AP_1_0, &err);
ck_assert_msg((intptr_t)dn, "error connecting to datanode: %s", err);
err = hdfs_datanode_read_file(dn, 0/*offset-in-block*/,
bl->ob_val._located_block._len,
ofd,
i*blocksz,
false/*crcs*/);
hdfs_datanode_delete(dn);
}
fail_if(err, "error reading block: %s", err);
}
end = _now();
fprintf(stderr, "Read %d MB to file in %ld ms%s, %02g MB/s\n",
towrite/1024/1024, end - begin, _i? " (with crcs)":"",
(double)towrite/(end-begin)/1024*1000/1024);
hdfs_object_free(bls);
fail_if(filecmp(fd, ofd, towrite), "read differed from write");
s = hdfs_delete(h, tf, false/*recurse*/, &e);
if (e)
fail("exception: %s", hdfs_exception_get_message(e));
ck_assert_msg(s, "delete returned false");
}
