bool downloadS3(const char *urlWithOptions) { if (!urlWithOptions) { return false; } int data_len = BUF_SIZE; char data_buf[BUF_SIZE]; bool ret = true; thread_setup(); GPReader *reader = reader_init(urlWithOptions); if (!reader) { return false; } do { data_len = BUF_SIZE; if (!reader_transfer_data(reader, data_buf, data_len)) { fprintf(stderr, "Failed to read data from Amazon S3\n"); ret = false; break; } fwrite(data_buf, (size_t)data_len, 1, stdout); } while (data_len && !S3QueryIsAbortInProgress()); reader_cleanup(&reader); thread_cleanup(); return ret; }
/* * Import data into GPDB. * invoked by GPDB, be careful with C++ exceptions. */ Datum s3_import(PG_FUNCTION_ARGS) { /* Must be called via the external table format manager */ if (!CALLED_AS_EXTPROTOCOL(fcinfo)) elog(ERROR, "extprotocol_import: not called by external protocol manager"); /* Get our internal description of the protocol */ GPReader *gpreader = (GPReader *)EXTPROTOCOL_GET_USER_CTX(fcinfo); /* last call. destroy reader */ if (EXTPROTOCOL_IS_LAST_CALL(fcinfo)) { thread_cleanup(); if (!reader_cleanup(&gpreader)) { ereport(ERROR, (0, errmsg("Failed to cleanup S3 extension: %s", gpReaderErrorMessage.c_str()))); } EXTPROTOCOL_SET_USER_CTX(fcinfo, NULL); PG_RETURN_INT32(0); } /* first call. do any desired init */ if (gpreader == NULL) { const char *url_with_options = EXTPROTOCOL_GET_URL(fcinfo); thread_setup(); gpreader = reader_init(url_with_options); if (!gpreader) { ereport(ERROR, (0, errmsg("Failed to init S3 extension, segid = %d, " "segnum = %d, please check your " "configurations and net connection: %s", s3ext_segid, s3ext_segnum, gpReaderErrorMessage.c_str()))); } check_essential_config(); EXTPROTOCOL_SET_USER_CTX(fcinfo, gpreader); } char *data_buf = EXTPROTOCOL_GET_DATABUF(fcinfo); int32 data_len = EXTPROTOCOL_GET_DATALEN(fcinfo); if (!reader_transfer_data(gpreader, data_buf, data_len)) { ereport(ERROR, (0, errmsg("s3_import: could not read data: %s", gpReaderErrorMessage.c_str()))); } PG_RETURN_INT32(data_len); }