/*
* Import data into GPDB.
* invoked by GPDB, be careful with C++ exceptions.
*/
Datum s3_import(PG_FUNCTION_ARGS) {
S3ExtBase *myData;
char *data;
int data_len;
size_t nread = 0;
/* Must be called via the external table format manager */
if (!CALLED_AS_EXTPROTOCOL(fcinfo))
elog(ERROR,
"extprotocol_import: not called by external protocol manager");
/* Get our internal description of the protocol */
myData = (S3ExtBase *)EXTPROTOCOL_GET_USER_CTX(fcinfo);
if (EXTPROTOCOL_IS_LAST_CALL(fcinfo)) {
if (myData) {
thread_cleanup();
if (!myData->Destroy()) {
ereport(ERROR, (0, errmsg("Failed to cleanup S3 extention")));
}
delete myData;
}
/*
* Cleanup function for the XML library.
*/
xmlCleanupParser();
PG_RETURN_INT32(0);
}
if (myData == NULL) {
/* first call. do any desired init */
curl_global_init(CURL_GLOBAL_ALL);
thread_setup();
const char *p_name = "s3";
char *url_with_options = EXTPROTOCOL_GET_URL(fcinfo);
char *url = truncate_options(url_with_options);
char *config_path = get_opt_s3(url_with_options, "config");
if (!config_path) {
// no config path in url, use default value
// data_folder/gpseg0/s3/s3.conf
config_path = strdup("s3/s3.conf");
}
bool result = InitConfig(config_path, "");
if (!result) {
free(config_path);
ereport(ERROR, (0, errmsg("Can't find config file, please check")));
} else {
ClearConfig();
free(config_path);
}
InitLog();
if (s3ext_accessid == "") {
ereport(ERROR, (0, errmsg("ERROR: access id is empty")));
}
if (s3ext_secret == "") {
ereport(ERROR, (0, errmsg("ERROR: secret is empty")));
}
if ((s3ext_segnum == -1) || (s3ext_segid == -1)) {
ereport(ERROR, (0, errmsg("ERROR: segment id is invalid")));
}
myData = CreateExtWrapper(url);
if (!myData ||
!myData->Init(s3ext_segid, s3ext_segnum, s3ext_chunksize)) {
if (myData) delete myData;
ereport(ERROR, (0, errmsg("Failed to init S3 extension, segid = "
"%d, segnum = %d, please check your "
"configurations and net connection",
s3ext_segid, s3ext_segnum)));
}
EXTPROTOCOL_SET_USER_CTX(fcinfo, myData);
free(url);
}
/* =======================================================================
* DO THE IMPORT
* =======================================================================
*/
data = EXTPROTOCOL_GET_DATABUF(fcinfo);
data_len = EXTPROTOCOL_GET_DATALEN(fcinfo);
uint64_t readlen = 0;
if (data_len > 0) {
readlen = data_len;
if (!myData->TransferData(data, readlen))
ereport(ERROR, (0, errmsg("s3_import: could not read data")));
nread = (size_t)readlen;
// S3DEBUG("read %d data from S3", nread);
}
PG_RETURN_INT32((int)nread);
}
TEST(S3Common, UrlOptions) {
EXPECT_STREQ("secret_test",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test",
"secret"));
EXPECT_STREQ(
"\".\\!@#$%^&*()DFGHJK\"",
get_opt_s3(
"s3://neverland.amazonaws.com accessid=\".\\!@#$%^&*()DFGHJK\"",
"accessid"));
EXPECT_STREQ("3456789",
get_opt_s3("s3://neverland.amazonaws.com chunksize=3456789",
"chunksize"));
EXPECT_STREQ(
"secret_test",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
"secret"));
EXPECT_STREQ(
"\".\\!@#$%^&*()DFGHJK\"",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
"accessid"));
EXPECT_STREQ(
"3456789",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
"chunksize"));
EXPECT_STREQ("secret_test",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever accessid=\".\\!@#$%^&*()DFGHJK\" "
"chunksize=3456789 KingOfTheWorld=sanpang",
"secret"));
EXPECT_STREQ("secret_test",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
"chunksize=3456789 KingOfTheWorld=sanpang",
"secret"));
EXPECT_STREQ("3456789",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld=sanpang ",
"chunksize"));
EXPECT_STREQ("3456789",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld=sanpang ",
"chunksize"));
EXPECT_STREQ("=sanpang",
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld==sanpang ",
"KingOfTheWorld"));
EXPECT_EQ((char *)NULL, get_opt_s3("", "accessid"));
EXPECT_EQ((char *)NULL, get_opt_s3(NULL, "accessid"));
EXPECT_EQ((char *)NULL,
get_opt_s3("s3://neverland.amazonaws.com", "secret"));
EXPECT_EQ((char *)NULL,
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever accessid= chunksize=3456789 "
"KingOfTheWorld=sanpang",
"accessid"));
EXPECT_EQ(
(char *)NULL,
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
""));
EXPECT_EQ(
(char *)NULL,
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
NULL));
EXPECT_EQ((char *)NULL,
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld=sanpang ",
"chunk size"));
}
TEST(Common, UrlOptions) {
char *option = NULL;
EXPECT_STREQ(
"secret_test",
option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test",
"secret"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"\".\\!@#$%^&*()DFGHJK\"",
option = get_opt_s3(
"s3://neverland.amazonaws.com accessid=\".\\!@#$%^&*()DFGHJK\"",
"accessid"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"3456789",
option = get_opt_s3("s3://neverland.amazonaws.com chunksize=3456789",
"chunksize"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ("secret_test",
option = get_opt_s3(
"s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
"secret"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ("\".\\!@#$%^&*()DFGHJK\"",
option = get_opt_s3(
"s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
"accessid"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ("3456789",
option = get_opt_s3(
"s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
"chunksize"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"secret_test",
option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever accessid=\".\\!@#$%^&*()DFGHJK\" "
"chunksize=3456789 KingOfTheWorld=sanpang",
"secret"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"secret_test",
option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
"chunksize=3456789 KingOfTheWorld=sanpang",
"secret"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"3456789",
option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld=sanpang ",
"chunksize"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"3456789",
option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld=sanpang ",
"chunksize"));
if (option) {
free(option);
option = NULL;
}
EXPECT_STREQ(
"=sanpang",
option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld==sanpang ",
"KingOfTheWorld"));
if (option) {
free(option);
option = NULL;
}
EXPECT_THROW(get_opt_s3("", "accessid"), std::runtime_error);
EXPECT_THROW(get_opt_s3(NULL, "accessid"), std::runtime_error);
EXPECT_THROW(get_opt_s3("s3://neverland.amazonaws.com", "secret"),
std::runtime_error);
EXPECT_THROW(get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever accessid= chunksize=3456789 "
"KingOfTheWorld=sanpang",
"accessid"),
std::runtime_error);
EXPECT_THROW(
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
""),
std::runtime_error);
EXPECT_THROW(
get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
NULL),
std::runtime_error);
EXPECT_THROW(get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
"chunksize=3456789 KingOfTheWorld=sanpang ",
"chunk size"),
std::runtime_error);
}
