TEST_F(TestHawqRegister, TestUsage2Case1FileUnderTableDirectory) {
SQLUtility util;
string test_root(util.getTestRootPath());
util.execute("drop table if exists t;");
util.execute("create table t(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/file_under_table_directory.yml", test_root.c_str()));
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/usage2case1/file_under_table_directory_tpl.yml", test_root.c_str()));
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid("t");
hawq::test::HdfsConfig hc;
string hdfs_prefix;
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testusage2case1fileUndertabledirectory.t", HAWQ_DB, t_yml.c_str())));
util.query("select * from t;", 100);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
}
TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
SQLUtility util;
string test_root(util.getTestRootPath());
std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"};
std::vector<string> fmt_matrix = {"row", "parquet"};
std::vector<string> option_matrix = {"--force", "-F"};
int suffix=0;
for (auto & opt : option_matrix) {
suffix = 0;
for (auto & ddl : create_table_matrix) {
for (auto & fmt : fmt_matrix) {
suffix++;
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_tpl_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
string t_yml_tpl_new(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_tpl_new_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
string t_yml_new(hawq::test::stringFormat("%s/ManagementTool/usage2case2/t_new_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
auto t = hawq::test::stringFormat("t_usage2_case2_%s", std::to_string(suffix).c_str());
auto nt = hawq::test::stringFormat("nt_usage2_case2_%s", std::to_string(suffix).c_str());
util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
util.execute(hawq::test::stringFormat("drop table if exists %s;", nt.c_str()));
// hawq register -d hawq_feature_test -c t_#.yml nt_usage2_case2_#
util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 100);", t.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 200);", t.c_str()));
util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 200);
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid(t);
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testusage2case2expected.%s", HAWQ_DB, t_yml.c_str(), nt.c_str())));
util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 200);
// hawq register --force/-F -d hawq_feature_test -c t_new_#.yml nt_usage2_case2_#
util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 50);", t.c_str()));
util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 50);
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID_OLD@"]= getTableOid(nt);
strs_src_dst["@TABLE_OID_NEW@"]= getTableOid(t);
frep.replace(t_yml_tpl_new, t_yml_new, strs_src_dst);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register %s -d %s -c %s testhawqregister_testusage2case2expected.%s", opt.c_str(), HAWQ_DB, t_yml_new.c_str(), nt.c_str())));
util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 150);
if (fmt == "row")
checkPgAOSegValue(nt, "-1", "aoseg");
else
checkPgAOSegValue(nt, "-1", "paqseg");
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml_new.c_str())));
util.execute(hawq::test::stringFormat("drop table %s;", t.c_str()));
util.execute(hawq::test::stringFormat("drop table %s;", nt.c_str()));
}
}
}
}
TEST_F(TestHawqRegister, TestUsage1EofFailure) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
/* register a parquet file with eof=-1, should success */
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e -1", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
/* register a parquet file with eof > filesize, should success */
int size = getFileSize(filePath.c_str());
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1);
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
/* register a parquet file with eof=float, should success */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e 11.1", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestUsage1FileNotExist) {
SQLUtility util;
util.execute("create table hawqregister(i int);");
util.query("select * from hawqregister;", 0);
auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestRollBackHDFSFilePathContainErrorSymbol) {
SQLUtility util;
string test_root(util.getTestRootPath());
util.execute("drop table if exists t;");
util.execute("drop table if exists nt;");
util.execute("create table t(i int) with (appendonly=true, orientation=row) distributed randomly;");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
util.execute("create table nt(i int) with (appendonly=true, orientation=row) distributed by (i);");
util.execute("insert into nt select generate_series(1, 100);");
util.query("select * from nt;", 100);
string t_yml(hawq::test::stringFormat("%s/ManagementTool/rollback/error_schema.yml", test_root.c_str()));
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/rollback/error_schema.yml", test_root.c_str()));
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid("t", "testhawqregister_testrollbackhdfsfilepathcontainerrorsymbol");
string hdfs_prefix;
hawq::test::HdfsConfig hc;
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testrollbackhdfsfilepathcontainerrorsymbol.nt", HAWQ_DB, t_yml.c_str())));
util.query("select * from t;", 100);
util.query("select * from nt;", 100);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
util.execute("drop table nt;");
}
void TestHawqRegister::runYamlCaseTableExists(std::string casename, std::string ymlname, int isexpectederror=1, int checknum=100) {
SQLUtility util;
string test_root(util.getTestRootPath());
util.execute("drop table if exists t;");
util.execute("drop table if exists nt;");
util.execute("create table t(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
util.execute("create table nt(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
util.execute("insert into nt select generate_series(1, 100);");
util.query("select * from nt;", 100);
string t_yml(hawq::test::stringFormat("%s/ManagementTool/%s.yml", test_root.c_str(), ymlname.c_str()));
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/%s_tpl.yml", test_root.c_str(), ymlname.c_str()));
hawq::test::FileReplace frep;
std::unordered_map<std::string, std::string> strs_src_dst;
strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
strs_src_dst["@TABLE_OID@"]= getTableOid("t");
hawq::test::HdfsConfig hc;
string hdfs_prefix;
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
EXPECT_EQ(isexpectederror, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str())));
if (isexpectederror > 0) {
util.query("select * from t;", 100);
util.query("select * from nt;", 100);
} else {
util.query("select * from nt;", checknum);
}
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
util.execute("drop table nt;");
}
TEST_F(TestHawqRegister, TestUsage2Case1EmptyTable) {
SQLUtility util;
util.execute("drop table if exists t9;");
util.execute("create table t9(i int) with (appendonly=true, orientation=row) distributed randomly;");
EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t9.yml testhawqregister_testusage2case1emptytable.t9"));
EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t9.yml testhawqregister_testusage2case1emptytable.nt9"));
util.query("select * from nt9;", 0);
std::string reloid = getTableOid("nt9");
/* An empty table has no row in pg_aoseg.pg_aoseg_xxx table */
util.query(hawq::test::stringFormat("select * from pg_aoseg.pg_aoseg_%s;", reloid.c_str()), 0);
EXPECT_EQ(0, Command::getCommandStatus("rm -rf t9.yml"));
util.execute("drop table t9;");
util.execute("drop table nt9;");
}
TEST_F(TestHawqRegister, TestUsage2Case1Expected) {
SQLUtility util;
string fmt_prefix;
std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"};
std::vector<string> fmt_matrix = {"row", "parquet"};
int suffix=0;
for (auto & ddl : create_table_matrix) {
for (auto & fmt : fmt_matrix) {
if (fmt.compare("row") == 0)
fmt_prefix = "aoseg";
else
fmt_prefix = "paqseg";
suffix++;
auto t = hawq::test::stringFormat("t_usage2_case1_%s", std::to_string(suffix).c_str());
auto nt = hawq::test::stringFormat("nt_usage2_case1_%s", std::to_string(suffix).c_str());
util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
util.execute(hawq::test::stringFormat("drop table if exists %s;", nt.c_str()));
// hawq register -d hawq_feature_test -c t_usage2_case1_#.yml nt_usage2_case1_#, where nt_usage2_case1_# does not exist
util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 100);", t.c_str()));
util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 100);
// get pg_aoseg.pg_xxxseg_xxx table
std::string reloid1 = getTableOid(t.c_str());
string result1 = util.getQueryResultSetString(hawq::test::stringFormat("select * from pg_aoseg.pg_%s_%s order by segno;", fmt_prefix.c_str(), reloid1.c_str()));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_%s.yml testhawqregister_testusage2case1expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), t.c_str())));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_%s.yml testhawqregister_testusage2case1expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), nt.c_str())));
util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 100);
// check pg_aoseg.pg_xxxseg_xxx table
std::string reloid2 = getTableOid(nt.c_str());
string result2 = util.getQueryResultSetString(hawq::test::stringFormat("select * from pg_aoseg.pg_%s_%s order by segno;", fmt_prefix.c_str(), reloid2.c_str()));
EXPECT_EQ(result1, result2);
// hawq register -d hawq_feature_test -c t_usage2_case1_#.yml nt_usage2_case1_#, where nt_usage2_case1_# exists
util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 150);", t.c_str()));
util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 50);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_%s.yml testhawqregister_testusage2case1expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), t.c_str())));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_%s.yml testhawqregister_testusage2case1expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), nt.c_str())));
util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 150);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf t_%s.yml", std::to_string(suffix).c_str())));
util.execute(hawq::test::stringFormat("drop table %s;", t.c_str()));
util.execute(hawq::test::stringFormat("drop table %s;", nt.c_str()));
}
}
}
TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
/* register a non-hdfs parquet file, should failed */
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, filePath.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
}
TEST_F(TestParquet, TestMultipleType) {
SQLUtility util;
util.execute("drop table if exists t1");
DataGenerator dGen(&util);
dGen.genTableWithFullTypes("t1", true, "parquet");
util.query(
"select * from t1",
"2147483647|||00:00:00||ff:89:71:45:ae:01|2000-01-01 "
"07:00:00+08||32767|t|192.168.1.255/"
"32|<(1,2),3>|[(0,0),(6,6)]|-178000000 "
"years|0|-$21,474,836.48|(200,400),(100,200)||<aa>bb</"
"aa>||123456789a|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
"64|||1|0|(1,2)|4277-12-31|128|\n0|((100,123),(5,10),(7,2),(4,5))|hello "
"world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
"8a2e:370:7344/128|<(1,2),3>|[(0,0),(6,6)]||1|$0.00|(2,3),(0,1)|hello "
"world|<aa>bb</aa>||aaaa|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
"64|0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n||abcd|15:01:03||"
"|2000-01-01 07:00:00+08|||t|||[(0,0),(6,6)]|-178000000 "
"years|0|-$21,474,836.48|(200,400),(100,200)||<aa>bb</"
"aa>||123456789a|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
"64|||1|0|(1,2)|4277-12-31|128|\n0|((100,123),(5,10),(7,2),(4,5))|hello "
"world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
"8a2e:370:7344/128|<(1,2),3>||||$0.00||hello "
"world|||aaaa|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
"64|0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n0|((100,123),(5,"
"10),(7,2),(4,5))|hello "
"world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
"8a2e:370:7344/128|<(1,2),3>||||$0.00||hello "
"world|||||0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n0|((100,"
"123),(5,10),(7,2),(4,5))|hello "
"world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
"8a2e:370:7344/128|<(1,2),3>||||||hello world|||||0||34|||||\n");
}
TEST_F(TestHawqRegister, TestUsage1NotParquetFile) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_not_paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test_not_paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_test_not_paq", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestDataTypes) {
SQLUtility util;
string rootPath(util.getTestRootPath());
/* This parquet file is generate by HIVE, using the table created by */
/* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */
string relativePath("/ManagementTool/test_hawq_register_data_types.paq");
string filePath = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 1);
util.execute("drop table hawqregister;");
}
TEST_F(TestHawqRegister, TestUsage1FolderFailure) {
SQLUtility util;
string folderName = "usage1tmp";
string folderNameNotExist = "usage1tmpNotExist";
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
string relativePath2("/ManagementTool/files_incomplete.yml");
string filePath2 = rootPath + relativePath2;
auto cmd = hawq::test::stringFormat("hdfs dfs -mkdir %s/usage1tmp", getHdfsLocation().c_str(), "");
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
/* register a empty folder, should success */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
/* register a not exist folder, should fail */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderNameNotExist.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
/* register a folder with eof, should fail */
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister -e 100", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
/* register a folder containing parquet and non-parquet files, should fail */
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/", filePath2.c_str(), getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
cmd = hawq::test::stringFormat("hdfs dfs -rm -r %s/%s", getHdfsLocation().c_str(), folderName.c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
}
TEST_F(TestParquet, TestCompression) {
SQLUtility util;
util.execute("drop table if exists t21");
util.execute("drop table if exists t22");
DataGenerator dGen(&util);
dGen.genTableWithNull("t21", true, "parquet", "gzip", 9);
dGen.genTableWithNull("t22", true, "parquet", "snappy");
util.query("select * from t21,t22",
"15||aa|15||aa|\n15||aa|||WET|\n15||aa||51||\n||WET|15||aa|\n||"
"WET|||WET|\n||WET||51||\n|51||15||aa|\n|51||||WET|\n|51|||51||"
"\n");
}
TEST_F(TestHawqRegister, TestFiles) {
SQLUtility util;
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath1 = rootPath + relativePath;
relativePath = "/ManagementTool/test_hawq_register_hive.paq";
string filePath2 = rootPath + relativePath;
auto cmd = hawq::test::stringFormat("hadoop fs -mkdir -p %s/hawq_register_test/t", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq1.paq", filePath1.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq2.paq", filePath1.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hawq.paq", filePath1.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive1.paq", filePath2.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive2.paq", filePath2.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hive.paq", filePath2.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister", HAWQ_DB, getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 12);
util.execute("insert into hawqregister values(1);");
util.query("select * from hawqregister;", 13);
cmd = hawq::test::stringFormat("hadoop fs -rm -r %s/hawq_register_test", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
TEST_F(TestQueryPrepare, TestPrepareParameters) {
SQLUtility util;
// prepare
util.execute("drop table if exists test1");
util.execute("drop table if exists test2");
util.execute("create table test1 ("
" unique1 int4,"
" unique2 int4,"
" two int4,"
" four int4,"
" ten int4,"
" twenty int4,"
" hundred int4,"
" thousand int4,"
" twothousand int4,"
" fivethous int4,"
" tenthous int4,"
" odd int4,"
" even int4,"
" stringu1 name,"
" stringu2 name,"
" string4 name) with oids");
util.execute("create table test2 ("
" name text,"
" thepath path)");
std::string pwd = util.getTestRootPath();
std::string cmd = "COPY test1 FROM '" + pwd + "/query/data/tenk.data'";
std::cout << cmd << std::endl;
util.execute(cmd);
cmd = "COPY test2 FROM '" + pwd + "/query/data/streets.data'";
std::cout << cmd << std::endl;
util.execute(cmd);
// do test
util.execSQLFile("query/sql/prepare-parameters.sql",
"query/ans/prepare-parameters.ans");
// cleanup
util.execute("drop table test1");
util.execute("drop table test2");
}
TEST_F(TestParquet, TestPartition) {
SQLUtility util;
util.execute("drop table if exists t4");
util.execute(
"create table t4 (id SERIAL,a1 int,a2 char(5)) WITH (appendonly=true, "
"orientation=parquet,compresstype=gzip,compresslevel=1) distributed "
"randomly Partition by range(a1) (start(1) end(16) every(8)"
"WITH(appendonly = true, orientation = parquet, compresstype = "
"snappy))");
util.execute("insert into t4(a1,a2) values(generate_series(1,5),'M')");
util.execute(
"alter table t4 add partition new_p start(17) end (20) WITH "
"(appendonly=true, orientation=parquet, compresstype=snappy)");
util.execute(
"alter table t4 add default partition df_p WITH "
"(appendonly=true, orientation=parquet, compresstype=gzip, "
"compresslevel=3)");
util.execute("insert into t4(a1,a2) values(generate_series(6,25),'F')");
util.query("select * from t4", 25);
}
TEST_F(TestHawqRegister, TestUsage2Case1ErrorHashTableRegistry) {
SQLUtility util;
util.execute("drop table if exists t_1_1;");
util.execute("drop table if exists t_1_2;");
util.execute("drop table if exists t_1_3;");
util.execute("drop table if exists t_2;");
util.execute("drop table if exists nt_1;");
util.execute("drop table if exists nt_2;");
util.execute("create table t_1_1(i int, j int, k int) with (appendonly=true, orientation=row, bucketnum=12) distributed by (i, j);");
util.execute("insert into t_1_1 select generate_series(1, 100);");
util.query("select * from t_1_1;", 100);
util.execute("create table t_1_2(i int, j int, k int) with (appendonly=true, orientation=row) distributed by (i);");
util.execute("insert into t_1_2 select generate_series(1, 100);");
util.query("select * from t_1_2;", 100);
util.execute("create table t_1_3(i int, j int, k int) with (appendonly=true, orientation=row) distributed randomly;");
util.execute("insert into t_1_3 select generate_series(1, 100);");
util.query("select * from t_1_3;", 100);
util.execute("create table t_2(i int, j int, k int) with (appendonly=true, orientation=row) distributed by (i);");
util.execute("insert into t_2 select generate_series(1, 100);");
util.query("select * from t_2;", 100);
util.execute("create table nt_1(i int, j int, k int) with (appendonly=true, orientation=row) distributed by (i, j);");
util.execute("insert into nt_1 select generate_series(1, 100);");
util.query("select * from nt_1;", 100);
util.execute("create table nt_2(i int, j int, k int) with (appendonly=true, orientation=row) distributed by (j);");
util.execute("insert into nt_2 select generate_series(1, 100);");
util.query("select * from nt_2;", 100);
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_1_1.yml testhawqregister_testusage2case1errorhashtableregistry.t_1_1", HAWQ_DB)));
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_1_1.yml testhawqregister_testusage2case1errorhashtableregistry.nt_1", HAWQ_DB)));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_1_2.yml testhawqregister_testusage2case1errorhashtableregistry.t_1_2", HAWQ_DB)));
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_1_2.yml testhawqregister_testusage2case1errorhashtableregistry.nt_1", HAWQ_DB)));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_1_3.yml testhawqregister_testusage2case1errorhashtableregistry.t_1_3", HAWQ_DB)));
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_1_3.yml testhawqregister_testusage2case1errorhashtableregistry.nt_1", HAWQ_DB)));
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_2.yml testhawqregister_testusage2case1errorhashtableregistry.t_2", HAWQ_DB)));
EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_2.yml testhawqregister_testusage2case1errorhashtableregistry.nt_2", HAWQ_DB)));
EXPECT_EQ(0, Command::getCommandStatus("rm -rf t_1_1.yml"));
EXPECT_EQ(0, Command::getCommandStatus("rm -rf t_1_2.yml"));
EXPECT_EQ(0, Command::getCommandStatus("rm -rf t_1_3.yml"));
EXPECT_EQ(0, Command::getCommandStatus("rm -rf t_2.yml"));
util.execute("drop table t_1_1;");
util.execute("drop table t_1_2;");
util.execute("drop table t_1_3;");
util.execute("drop table t_2;");
util.execute("drop table nt_1;");
util.execute("drop table nt_2;");
}
TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
// Register file/folder into HAWQ by specific file/folder name
SQLUtility util;
string rootPath(util.getTestRootPath());
string filePath = rootPath + "/ManagementTool/data/parquet200/dat.paq";
string folderPath = rootPath + "/ManagementTool/data/parquet200sum/";
vector<string> ddl_orientation_matrix = {"parquet"};
vector<string> distribution_policy_matrix = {"", "DISTRIBUTED RANDOMLY"};
vector<string> folder_matrix = {"/usage1tmp/", "/usage1tmp"};
for(int i = 0; i < ddl_orientation_matrix.size() * distribution_policy_matrix.size() * 2 + ddl_orientation_matrix.size() * distribution_policy_matrix.size() * folder_matrix.size(); ++i) {
util.execute(hawq::test::stringFormat("drop table if exists t_%s;", std::to_string(i).c_str()));
}
auto register_lambda = [&] () {
int suffix = 0;
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1dat.paq t_#
for(auto & ddl : ddl_orientation_matrix) {
for(auto & policy : distribution_policy_matrix) {
auto cmd = hawq::test::stringFormat("hdfs dfs -put -f %s %s/usage1dat.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 200);
util.execute(hawq::test::stringFormat("insert into t_%s values(201);", std::to_string(suffix).c_str()));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 201);
suffix ++;
}
}
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1dat.paq -e eof t_#
for(auto & ddl : ddl_orientation_matrix) {
for(auto & policy : distribution_policy_matrix) {
auto cmd = hawq::test::stringFormat("hdfs dfs -put -f %s %s/usage1dat.paq", filePath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq -e 596 t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 100);
util.execute(hawq::test::stringFormat("insert into t_%s values(101);", std::to_string(suffix).c_str()));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 101);
suffix ++;
}
}
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1tmp/ t_#
// hawq register -d hawq_feature_test -f hdfs://localhost:8020/usage1tmp t_#
for(auto & ddl : ddl_orientation_matrix) {
for(auto & policy : distribution_policy_matrix) {
for(auto & folder : folder_matrix) {
auto cmd = hawq::test::stringFormat("hdfs dfs -mkdir -p %s/usage1tmp/", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
cmd = hawq::test::stringFormat("hdfs dfs -put -f %s/*.paq %s/usage1tmp/", folderPath.c_str(), getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0);
cmd = hawq::test::stringFormat("hawq register -d %s -f %s%s t_%s", HAWQ_DB, getHdfsLocation().c_str(), folder.c_str(), std::to_string(suffix).c_str());
EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 200);
util.execute(hawq::test::stringFormat("insert into t_%s values(201);", std::to_string(suffix).c_str()));
util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 201);
suffix ++;
}
}
}
}; // register_lambda
auto gc_lambda = [&] () {
auto sql = hawq::test::stringFormat("hdfs dfs -rm -r %s/usage1tmp/", getHdfsLocation().c_str());
EXPECT_EQ(0, Command::getCommandStatus(sql));
for(int i = 0; i < ddl_orientation_matrix.size() * distribution_policy_matrix.size() * 2 + ddl_orientation_matrix.size() * distribution_policy_matrix.size() * folder_matrix.size(); ++i) {
util.execute(hawq::test::stringFormat("drop table t_%s;", std::to_string(i).c_str()));
}
};
register_lambda();
gc_lambda();
}
TEST_F(TestParquet, TestSize) {
SQLUtility util;
// value/record size equal to pagesize/rowgroupsize
util.execute("drop table if exists t31");
util.execute(
"create table t31 (a1 char(10485760), a2 char(10485760), a3 "
"char(10485760), a4 char(10485760), a5 char(10485760), a6 "
"char(10485760), a7 char(10485760), a8 char(10485760), a9 "
"char(10485760), a10 char(10485760)) with(appendonly=true, "
"orientation=parquet, pagesize=10485760, rowgroupsize=104857600)");
util.execute(
"insert into t31 values ( ('a'::char(10485760)), "
"('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)), "
"('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)), "
"('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)) );",
false);
EXPECT_STREQ("ERROR: value for column \"a1\" exceeds pagesize 10485760!",
util.getPSQL()->getLastResult().substr(0, 56).c_str());
// single column, one data page contains several values, one rwo group
// contains several groups
util.execute("drop table if exists t32");
util.execute("drop table if exists t33");
util.execute(
"create table t32 ( a1 text ) with(appendonly=true, "
"orientation=parquet)");
util.execute("insert into t32 values(repeat('parquet',100))");
util.execute("insert into t32 values(repeat('parquet',20))");
util.execute("insert into t32 values(repeat('parquet',30))");
util.execute(
"create table t33 ( a1 text ) with(appendonly=true, orientation=parquet, "
"pagesize=1024, rowgroupsize=1025)");
util.execute("insert into t33 select * from t32");
util.query("select * from t33", 3);
// large data insert, several column values in one page, several rows in one
// rowgroup
util.execute("drop table if exists t34");
util.execute("drop table if exists t35");
util.execute(
"create table t34 (a1 char(1048576), a2 char(2048576), a3 "
"char(3048576), a4 char(4048576), a5 char(5048576), a6 char(6048576), a7 "
"char(7048576), a8 char(8048576), a9 char(9048576), a10 char(9)) "
"with(appendonly=true, orientation=parquet, pagesize=10485760, "
"rowgroupsize=90874386)");
util.execute(
"insert into t34 values ( ('a'::char(1048576)), "
"('a'::char(2048576)), ('a'::char(3048576)), ('a'::char(4048576)), "
"('a'::char(5048576)), ('a'::char(6048576)), ('a'::char(7048576)), "
"('a'::char(8048576)), ('a'::char(9048576)), ('a'::char(9)) )");
util.execute(
"create table t35 (a1 char(1048576), a2 char(2048576), a3 "
"char(3048576), a4 char(4048576), a5 char(5048576), a6 char(6048576), a7 "
"char(7048576), a8 char(8048576), a9 char(9048576), a10 char(9)) "
"with(appendonly=true, orientation=parquet, pagesize=10485760, "
"rowgroupsize=17437200)");
util.execute("insert into t35 select * from t34");
util.query("select * from t35", 1);
}