TEST(ExtWrapper, ValidateURL_useast1) {
S3ExtBase *myData;
myData = new S3Reader(
"s3://s3-us-east-1.amazonaws.com/s3test.pivotal.io/dataset1/normal");
ASSERT_TRUE(myData->ValidateURL());
EXPECT_STREQ("external-1", myData->get_region().c_str());
delete myData;
}
TEST(ExtWrapper, ValidateURL_apnortheast21) {
S3ExtBase *myData;
myData = new S3Reader(
"s3://s3-ap-northeast-2.amazonaws.com/s3test.pivotal.io/dataset1/"
"normal");
ASSERT_TRUE(myData->ValidateURL());
EXPECT_STREQ("ap-northeast-2", myData->get_region().c_str());
delete myData;
}
void ExtWrapperTest(const char *url, uint64_t buffer_size, const char *md5_str,
int segid, int segnum, uint64_t chunksize) {
InitConfig("test/s3.conf", "default");
s3ext_segid = segid;
s3ext_segnum = segnum;
MD5Calc m;
S3ExtBase *myData;
uint64_t nread = 0;
uint64_t buf_len = buffer_size;
char *buf = (char *)malloc(buffer_size);
ASSERT_NE((void *)NULL, buf);
if (strncmp(url, "http://localhost/", 17) == 0) {
myData = new S3Reader_fake(url);
} else {
myData = new S3Reader(url);
}
ASSERT_NE((void *)NULL, myData);
ASSERT_TRUE(myData->Init(segid, segnum, chunksize));
while (1) {
nread = buf_len;
ASSERT_TRUE(myData->TransferData(buf, nread));
if (nread == 0) break;
m.Update(buf, nread);
}
EXPECT_STREQ(md5_str, m.Get());
myData->Destroy();
delete myData;
free(buf);
}
/*
* Import data into GPDB.
* invoked by GPDB, be careful with C++ exceptions.
*/
Datum s3_import(PG_FUNCTION_ARGS) {
S3ExtBase *myData;
char *data;
int data_len;
size_t nread = 0;
/* Must be called via the external table format manager */
if (!CALLED_AS_EXTPROTOCOL(fcinfo))
elog(ERROR,
"extprotocol_import: not called by external protocol manager");
/* Get our internal description of the protocol */
myData = (S3ExtBase *)EXTPROTOCOL_GET_USER_CTX(fcinfo);
if (EXTPROTOCOL_IS_LAST_CALL(fcinfo)) {
if (myData) {
thread_cleanup();
if (!myData->Destroy()) {
ereport(ERROR, (0, errmsg("Failed to cleanup S3 extention")));
}
delete myData;
}
/*
* Cleanup function for the XML library.
*/
xmlCleanupParser();
PG_RETURN_INT32(0);
}
if (myData == NULL) {
/* first call. do any desired init */
curl_global_init(CURL_GLOBAL_ALL);
thread_setup();
const char *p_name = "s3";
char *url_with_options = EXTPROTOCOL_GET_URL(fcinfo);
char *url = truncate_options(url_with_options);
char *config_path = get_opt_s3(url_with_options, "config");
if (!config_path) {
// no config path in url, use default value
// data_folder/gpseg0/s3/s3.conf
config_path = strdup("s3/s3.conf");
}
bool result = InitConfig(config_path, "");
if (!result) {
free(config_path);
ereport(ERROR, (0, errmsg("Can't find config file, please check")));
} else {
ClearConfig();
free(config_path);
}
InitLog();
if (s3ext_accessid == "") {
ereport(ERROR, (0, errmsg("ERROR: access id is empty")));
}
if (s3ext_secret == "") {
ereport(ERROR, (0, errmsg("ERROR: secret is empty")));
}
if ((s3ext_segnum == -1) || (s3ext_segid == -1)) {
ereport(ERROR, (0, errmsg("ERROR: segment id is invalid")));
}
myData = CreateExtWrapper(url);
if (!myData ||
!myData->Init(s3ext_segid, s3ext_segnum, s3ext_chunksize)) {
if (myData) delete myData;
ereport(ERROR, (0, errmsg("Failed to init S3 extension, segid = "
"%d, segnum = %d, please check your "
"configurations and net connection",
s3ext_segid, s3ext_segnum)));
}
EXTPROTOCOL_SET_USER_CTX(fcinfo, myData);
free(url);
}
/* =======================================================================
* DO THE IMPORT
* =======================================================================
*/
data = EXTPROTOCOL_GET_DATABUF(fcinfo);
data_len = EXTPROTOCOL_GET_DATALEN(fcinfo);
uint64_t readlen = 0;
if (data_len > 0) {
readlen = data_len;
if (!myData->TransferData(data, readlen))
ereport(ERROR, (0, errmsg("s3_import: could not read data")));
nread = (size_t)readlen;
// S3DEBUG("read %d data from S3", nread);
}
PG_RETURN_INT32((int)nread);
}
