TEST(S3Common, TruncateOptions) {
EXPECT_STREQ(
"s3://neverland.amazonaws.com",
truncate_options("s3://neverland.amazonaws.com secret=secret_test"));
EXPECT_STREQ(
"s3://neverland.amazonaws.com",
truncate_options(
"s3://neverland.amazonaws.com accessid=\".\\!@#$%^&*()DFGHJK\""));
EXPECT_STREQ(
"s3://neverland.amazonaws.com",
truncate_options("s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789"));
EXPECT_STREQ(
"s3://neverland.amazonaws.com",
truncate_options("s3://neverland.amazonaws.com secret=secret_test "
"blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
"chunksize=3456789 KingOfTheWorld=sanpang"));
}
TEST(Common, TruncateOptions) {
char *truncated = NULL;
EXPECT_STREQ("s3://neverland.amazonaws.com",
truncated = truncate_options(
"s3://neverland.amazonaws.com secret=secret_test"));
if (truncated) {
free(truncated);
truncated = NULL;
}
EXPECT_STREQ(
"s3://neverland.amazonaws.com",
truncated = truncate_options(
"s3://neverland.amazonaws.com accessid=\".\\!@#$%^&*()DFGHJK\""));
if (truncated) {
free(truncated);
truncated = NULL;
}
EXPECT_STREQ("s3://neverland.amazonaws.com",
truncated = truncate_options(
"s3://neverland.amazonaws.com secret=secret_test "
"accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789"));
if (truncated) {
free(truncated);
truncated = NULL;
}
EXPECT_STREQ("s3://neverland.amazonaws.com",
truncated = truncate_options(
"s3://neverland.amazonaws.com secret=secret_test "
"blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
"chunksize=3456789 KingOfTheWorld=sanpang"));
if (truncated) {
free(truncated);
truncated = NULL;
}
}
/*
* Import data into GPDB.
* invoked by GPDB, be careful with C++ exceptions.
*/
Datum s3_import(PG_FUNCTION_ARGS) {
S3ExtBase *myData;
char *data;
int data_len;
size_t nread = 0;
/* Must be called via the external table format manager */
if (!CALLED_AS_EXTPROTOCOL(fcinfo))
elog(ERROR,
"extprotocol_import: not called by external protocol manager");
/* Get our internal description of the protocol */
myData = (S3ExtBase *)EXTPROTOCOL_GET_USER_CTX(fcinfo);
if (EXTPROTOCOL_IS_LAST_CALL(fcinfo)) {
if (myData) {
thread_cleanup();
if (!myData->Destroy()) {
ereport(ERROR, (0, errmsg("Failed to cleanup S3 extention")));
}
delete myData;
}
/*
* Cleanup function for the XML library.
*/
xmlCleanupParser();
PG_RETURN_INT32(0);
}
if (myData == NULL) {
/* first call. do any desired init */
curl_global_init(CURL_GLOBAL_ALL);
thread_setup();
const char *p_name = "s3";
char *url_with_options = EXTPROTOCOL_GET_URL(fcinfo);
char *url = truncate_options(url_with_options);
char *config_path = get_opt_s3(url_with_options, "config");
if (!config_path) {
// no config path in url, use default value
// data_folder/gpseg0/s3/s3.conf
config_path = strdup("s3/s3.conf");
}
bool result = InitConfig(config_path, "");
if (!result) {
free(config_path);
ereport(ERROR, (0, errmsg("Can't find config file, please check")));
} else {
ClearConfig();
free(config_path);
}
InitLog();
if (s3ext_accessid == "") {
ereport(ERROR, (0, errmsg("ERROR: access id is empty")));
}
if (s3ext_secret == "") {
ereport(ERROR, (0, errmsg("ERROR: secret is empty")));
}
if ((s3ext_segnum == -1) || (s3ext_segid == -1)) {
ereport(ERROR, (0, errmsg("ERROR: segment id is invalid")));
}
myData = CreateExtWrapper(url);
if (!myData ||
!myData->Init(s3ext_segid, s3ext_segnum, s3ext_chunksize)) {
if (myData) delete myData;
ereport(ERROR, (0, errmsg("Failed to init S3 extension, segid = "
"%d, segnum = %d, please check your "
"configurations and net connection",
s3ext_segid, s3ext_segnum)));
}
EXTPROTOCOL_SET_USER_CTX(fcinfo, myData);
free(url);
}
/* =======================================================================
* DO THE IMPORT
* =======================================================================
*/
data = EXTPROTOCOL_GET_DATABUF(fcinfo);
data_len = EXTPROTOCOL_GET_DATALEN(fcinfo);
uint64_t readlen = 0;
if (data_len > 0) {
readlen = data_len;
if (!myData->TransferData(data, readlen))
ereport(ERROR, (0, errmsg("s3_import: could not read data")));
nread = (size_t)readlen;
// S3DEBUG("read %d data from S3", nread);
}
PG_RETURN_INT32((int)nread);
}