TEST_F(TestHawqRegister, TestUsage1NotParquetTable) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
/* register a parquet file to a row random table, should fail */
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
/* register a parquet file to a row hash table, should fail */
util.execute("create table hawqregister(i int) distributed by (i);");
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
/* register a parquet file to a parquet hash table, should fail */
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
}
void TestHawqRegister::runYamlCaseTableNotExists(std::string casename, std::string ymlname, int isexpectederror=1, int checknum=100) {
SQLUtility util;
string test_root(util.getTestRootPath());
util.execute("drop table if exists t;");
util.execute("drop table if exists nt;");
util.execute("create table t(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
string t_yml(hawq::test::stringFormat("%s/ManagementTool/%s.yml", test_root.c_str(), ymlname.c_str()));
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/%s_tpl.yml", test_root.c_str(), ymlname.c_str()));
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid("t");
hawq::test::HdfsConfig hc;
string hdfs_prefix;
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(isexpectederror, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str())));
if (isexpectederror > 0) {
util.query("select * from t;", 100);
util.query("select * from pg_class where relname = 'nt';", 0);
} else {
util.query("select * from nt;", checknum);
}
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
util.execute("drop table if exists nt;");
}
TEST_F(TestHawqRegister, TestUsage1EofFailure) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
/* register a parquet file with eof=-1, should success */
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e -1", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
/* register a parquet file with eof > filesize, should success */
int size = getFileSize(filePath.c_str());
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1);
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
/* register a parquet file with eof=float, should success */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e 11.1", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestUsage2Case1FileUnderTableDirectory) {
SQLUtility util;
string test_root(util.getTestRootPath());
util.execute("drop table if exists t;");
util.execute("create table t(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/file_under_table_directory.yml", test_root.c_str()));
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/usage2case1/file_under_table_directory_tpl.yml", test_root.c_str()));
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid("t");
hawq::test::HdfsConfig hc;
string hdfs_prefix;
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testusage2case1fileUndertabledirectory.t", HAWQ_DB, t_yml.c_str())));
util.query("select * from t;", 100);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
}
TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
SQLUtility util;
string test_root(util.getTestRootPath());
std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"};
std::vector<string> fmt_matrix = {"row", "parquet"};
std::vector<string> option_matrix = {"--force", "-F"};
int suffix=0;
for (auto & opt : option_matrix) {
suffix = 0;
for (auto & ddl : create_table_matrix) {
for (auto & fmt : fmt_matrix) {
suffix++;
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_tpl_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
string t_yml_tpl_new(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_tpl_new_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
string t_yml_new(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_new_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
auto t = hawq::test::stringFormat("t_usage2_case2_%s", std::to_string(suffix).c_str());
auto nt = hawq::test::stringFormat("nt_usage2_case2_%s", std::to_string(suffix).c_str());
util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
util.execute(hawq::test::stringFormat("drop table if exists %s;", nt.c_str()));
// hawq register -d hawq_feature_test -c t_#.yml nt_usage2_case2_#
util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 100);", t.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 200);", t.c_str()));
util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 200);
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid(t);
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testusage2case2expected.%s", HAWQ_DB, t_yml.c_str(), nt.c_str())));
util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 200);
// hawq register --force/-F -d hawq_feature_test -c t_new_#.yml nt_usage2_case2_#
util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 50);", t.c_str()));
util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 50);
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID_OLD@"]= getTableOid(nt);
strs_src_dst["@TABLE_OID_NEW@"]= getTableOid(t);
frep.replace(t_yml_tpl_new, t_yml_new, strs_src_dst);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register %s -d %s -c %s testhawqregister_testusage2case2expected.%s", opt.c_str(), HAWQ_DB, t_yml_new.c_str(), nt.c_str())));
util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 150);
if (fmt == "row")
checkPgAOSegValue(nt, "-1", "aoseg");
else
checkPgAOSegValue(nt, "-1", "paqseg");
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml_new.c_str())));
util.execute(hawq::test::stringFormat("drop table %s;", t.c_str()));
util.execute(hawq::test::stringFormat("drop table %s;", nt.c_str()));
}
}
}
}
TEST_F(TestHawqRegister, TestRollBackHDFSFilePathContainErrorSymbol) {
SQLUtility util;
string test_root(util.getTestRootPath());
util.execute("drop table if exists t;");
util.execute("drop table if exists nt;");
util.execute("create table t(i int) with (appendonly=true, orientation=row) distributed randomly;");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
util.execute("create table nt(i int) with (appendonly=true, orientation=row) distributed by (i);");
util.execute("insert into nt select generate_series(1, 100);");
util.query("select * from nt;", 100);
string t_yml(hawq::test::stringFormat("%s/ManagementTool/rollback/error_schema.yml", test_root.c_str()));
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/rollback/error_schema.yml", test_root.c_str()));
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid("t", "testhawqregister_testrollbackhdfsfilepathcontainerrorsymbol");
string hdfs_prefix;
hawq::test::HdfsConfig hc;
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testrollbackhdfsfilepathcontainerrorsymbol.nt", HAWQ_DB, t_yml.c_str())));
util.query("select * from t;", 100);
util.query("select * from nt;", 100);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
util.execute("drop table nt;");
}
TEST_F(TestHawqRegister, TestUsage2Case1IncludeDirectory) {
SQLUtility util;
string test_root(util.getTestRootPath());
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put -f %s/ManagementTool/usage2case1 %s/", test_root.c_str(), getHdfsLocation().c_str())));
string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/includedirectory.yml", test_root.c_str()));
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testusage2case1includedirectory.nt", HAWQ_DB, t_yml.c_str())));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -rm -r %s/usage2case1", getHdfsLocation().c_str())));
}
TEST_F(TestHawqRegister, TestUsage1NoTable) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
/* register a parquet file to a table not exist, should fail */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
}
TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) {
SQLUtility util;
string filePath = util.getTestRootPath() + "/ManagementTool/";
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_pagesize.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_rowgroupsize.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_filesize.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_schema.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_checksum.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_dfs_url.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_bucketnum.yml testhawqregister_testusage2case1incorrectyaml.xx"));
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_constraint_partition.yml testhawqregister_testusage2case1incorrectyaml.xx"));
}
TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
/* register a non-hdfs parquet file, should failed */
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, filePath.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
}
TEST_F(TestExternalOid, TestExternalOidAll) {
SQLUtility util;
FileReplace frep;
auto test_root = util.getTestRootPath();
std::cout << test_root << std::endl;
std::unordered_map<std::string, std::string> D;
D["@SHARE_LIBRARY_PATH@"] = test_root + "/ExternalSource/lib/function.so";
D["@abs_datadir@"] = test_root + "/ExternalSource/data";
frep.replace(test_root + "/ExternalSource/sql/external_oid.sql.source",
test_root + "/ExternalSource/sql/external_oid.sql",
D);
frep.replace(test_root + "/ExternalSource/ans/external_oid.ans.source",
test_root + "/ExternalSource/ans/external_oid.ans",
D);
util.execSQLFile("ExternalSource/sql/external_oid.sql",
"ExternalSource/ans/external_oid.ans");
}
TEST_F(TestQueryPrepare, TestPrepareParameters) {
SQLUtility util;
// prepare
util.execute("drop table if exists test1");
util.execute("drop table if exists test2");
util.execute("create table test1 ("
" unique1 int4,"
" unique2 int4,"
" two int4,"
" four int4,"
" ten int4,"
" twenty int4,"
" hundred int4,"
" thousand int4,"
" twothousand int4,"
" fivethous int4,"
" tenthous int4,"
" odd int4,"
" even int4,"
" stringu1 name,"
" stringu2 name,"
" string4 name) with oids");
util.execute("create table test2 ("
" name text,"
" thepath path)");
std::string pwd = util.getTestRootPath();
std::string cmd = "COPY test1 FROM '" + pwd + "/query/data/tenk.data'";
std::cout << cmd << std::endl;
util.execute(cmd);
cmd = "COPY test2 FROM '" + pwd + "/query/data/streets.data'";
std::cout << cmd << std::endl;
util.execute(cmd);
// do test
util.execSQLFile("query/sql/prepare-parameters.sql",
"query/ans/prepare-parameters.ans");
// cleanup
util.execute("drop table test1");
util.execute("drop table test2");
}
TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hive.paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hive.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 1);
util.execute("insert into hawqregister values(1);");
util.query("select * from hawqregister;", 2);
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestDataTypes) {
SQLUtility util;
string rootPath(util.getTestRootPath());
/* This parquet file is generate by HIVE, using the table created by */
/* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */
string relativePath("/ManagementTool/test_hawq_register_data_types.paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 1);
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestUsage1FolderFailure) {
SQLUtility util;
string folderName = "usage1tmp";
string folderNameNotExist = "usage1tmpNotExist";
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
string relativePath2("/ManagementTool/files_incomplete.yml");
string filePath2 = rootPath + relativePath2;
auto cmd = hawq::test::stringFormat("hdfs dfs -mkdir %s/usage1tmp", getHdfsLocation().c_str(), "");
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
/* register a empty folder, should success */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
/* register a not exist folder, should fail */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderNameNotExist.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
/* register a folder with eof, should fail */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister -e 100", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
/* register a folder containing parquet and non-parquet files, should fail */
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/", filePath2.c_str(), getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
cmd = hawq::test::stringFormat("hdfs dfs -rm -r %s/%s", getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
}
TEST_F(TestExternalTable, DISABLED_TestExternalTableAll) {
SQLUtility util;
auto test_root = util.getTestRootPath();
auto replace_lambda = [&] () {
FileReplace frep;
std::unordered_map<std::string, std::string> D;
D["@hostname@"] = std::string("localhost");
D["@abs_srcdir@"] = test_root + "/ExternalSource";
D["@gpwhich_gpfdist@"] = std::string(std::string(getenv("GPHOME")) + "/bin/gpfdist");
frep.replace(test_root + "/ExternalSource/sql/exttab1.sql.source",
test_root + "/ExternalSource/sql/exttab1.sql",
D);
frep.replace(test_root + "/ExternalSource/ans/exttab1.ans.source",
test_root + "/ExternalSource/ans/exttab1.ans",
D);
};
replace_lambda();
util.execSQLFile("ExternalSource/sql/exttab1.sql",
"ExternalSource/ans/exttab1.ans");
}
TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) {
SQLUtility util;
string filePath = util.getTestRootPath() + "/ManagementTool/";
EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "files_incomplete.yml testhawqregister_testusage2case1mismatchfilenumber.xx"));
}
TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
// Register file/folder into HAWQ by specific file/folder name
SQLUtility util;
string rootPath(util.getTestRootPath());
string filePath = rootPath + "/ManagementTool/data/parquet200/dat.paq";
string folderPath = rootPath + "/ManagementTool/data/parquet200sum/";
vector<string> ddl_orientation_matrix = {"parquet"};
vector<string> distribution_policy_matrix = {"", "DISTRIBUTED RANDOMLY"};
vector<string> folder_matrix = {"/usage1tmp/", "/usage1tmp"};
for(int i = 0; i < ddl_orientation_matrix.size() * distribution_policy_matrix.size() * 2 + ddl_orientation_matrix.size() * distribution_policy_matrix.size() * folder_matrix.size(); ++i) {
util.execute(hawq::test::stringFormat("drop table if exists t_%s;", std::to_string(i).c_str()));
}
auto register_lambda = [&] () {
int suffix = 0;
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1dat.paq t_#
for(auto & ddl : ddl_orientation_matrix) {
for(auto & policy : distribution_policy_matrix) {
auto cmd = hawq::test::stringFormat("hdfs dfs -put -f %s %s/usage1dat.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 200);
util.execute(hawq::test::stringFormat("insert into t_%s values(201);", std::to_string(suffix).c_str()));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 201);
suffix ++;
}
}
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1dat.paq -e eof t_#
for(auto & ddl : ddl_orientation_matrix) {
for(auto & policy : distribution_policy_matrix) {
auto cmd = hawq::test::stringFormat("hdfs dfs -put -f %s %s/usage1dat.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq -e 596 t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 100);
util.execute(hawq::test::stringFormat("insert into t_%s values(101);", std::to_string(suffix).c_str()));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 101);
suffix ++;
}
}
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1tmp/ t_#
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1tmp t_#
for(auto & ddl : ddl_orientation_matrix) {
for(auto & policy : distribution_policy_matrix) {
for(auto & folder : folder_matrix) {
auto cmd = hawq::test::stringFormat("hdfs dfs -mkdir -p %s/usage1tmp/", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hdfs dfs -put -f %s/*.paq %s/usage1tmp/", folderPath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s%s t_%s", HAWQ_DB, getHdfsLocation().c_str(), folder.c_str(), std::to_string(suffix).c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 200);
util.execute(hawq::test::stringFormat("insert into t_%s values(201);", std::to_string(suffix).c_str()));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 201);
suffix ++;
}
}
}
}; // register_lambda
auto gc_lambda = [&] () {
auto sql = hawq::test::stringFormat("hdfs dfs -rm -r %s/usage1tmp/", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(sql));
for(int i = 0; i < ddl_orientation_matrix.size() * distribution_policy_matrix.size() * 2 + ddl_orientation_matrix.size() * distribution_policy_matrix.size() * folder_matrix.size(); ++i) {
util.execute(hawq::test::stringFormat("drop table t_%s;", std::to_string(i).c_str()));
}
};
register_lambda();
gc_lambda();
}