bool downloadS3(const char *urlWithOptions) {
if (!urlWithOptions) {
return false;
}
int data_len = BUF_SIZE;
char data_buf[BUF_SIZE];
bool ret = true;
thread_setup();
GPReader *reader = reader_init(urlWithOptions);
if (!reader) {
return false;
}
do {
data_len = BUF_SIZE;
if (!reader_transfer_data(reader, data_buf, data_len)) {
fprintf(stderr, "Failed to read data from Amazon S3\n");
ret = false;
break;
}
fwrite(data_buf, (size_t)data_len, 1, stdout);
} while (data_len && !S3QueryIsAbortInProgress());
reader_cleanup(&reader);
thread_cleanup();
return ret;
}
void S3KeyWriter::checkQueryCancelSignal() {
if (S3QueryIsAbortInProgress() && !this->uploadId.empty()) {
// to avoid dead-lock when other upload threads hold the lock
pthread_mutex_unlock(&this->mutex);
// wait for all threads to complete
for (size_t i = 0; i < threadList.size(); i++) {
pthread_join(threadList[i], NULL);
}
this->threadList.clear();
// to avoid double unlock as other parts may lock it
pthread_mutex_lock(&this->mutex);
S3DEBUG("Start aborting multipart uploading (uploadID: %s, %lu parts uploaded)",
this->uploadId.c_str(), this->etagList.size());
this->s3Interface->abortUpload(this->params.getS3Url(), this->uploadId);
S3DEBUG("Finished aborting multipart uploading (uploadID: %s)", this->uploadId.c_str());
this->etagList.clear();
this->uploadId.clear();
S3_DIE(S3QueryAbort, "Uploading is interrupted");
}
}
// curl's headers write function callback.
size_t RESTfulServiceHeadersWriteFuncCallback(char *ptr, size_t size, size_t nmemb, void *userp) {
if (S3QueryIsAbortInProgress()) {
return 0;
}
size_t realsize = size * nmemb;
Response *resp = (Response *)userp;
resp->appendHeadersBuffer(ptr, realsize);
return realsize;
}
int main(int argc, char *argv[]) {
bool ret = true;
s3ext_loglevel = EXT_ERROR;
s3ext_logtype = STDERR_LOG;
if (argc == 1) {
printUsage(stderr);
exit(EXIT_FAILURE);
}
/* Prepare to receive interrupts */
registerSignalHandler();
map<char, string> optionPairs = parseCommandLineArgs(argc, argv);
validateCommandLineArgs(optionPairs);
if (!optionPairs.empty()) {
const char *arg = optionPairs.begin()->second.c_str();
switch (optionPairs.begin()->first) {
case 'c':
ret = checkConfig(arg);
break;
case 'd':
ret = downloadS3(arg);
break;
case 'u':
case 'f':
ret = uploadS3(optionPairs['u'].c_str(), optionPairs['f'].c_str());
break;
case 'h':
printUsage(stdout);
break;
case 't':
printTemplate();
break;
default:
printUsage(stderr);
exit(EXIT_FAILURE);
}
}
// Abort should not print the failed info
if (ret || S3QueryIsAbortInProgress()) {
exit(EXIT_SUCCESS);
} else {
fprintf(stderr, "Failed. Please check the arguments and configuration file.\n\n");
printUsage(stderr);
exit(EXIT_FAILURE);
}
}
bool uploadS3(const char *urlWithOptions, const char *fileToUpload) {
if (!urlWithOptions) {
return false;
}
size_t data_len = BUF_SIZE;
char data_buf[BUF_SIZE];
size_t read_len = 0;
bool ret = true;
thread_setup();
GPWriter *writer = writer_init(urlWithOptions);
if (!writer) {
return false;
}
FILE *fd = fopen(fileToUpload, "r");
if (fd == NULL) {
fprintf(stderr, "File does not exist\n");
ret = false;
} else {
do {
read_len = fread(data_buf, 1, data_len, fd);
if (read_len == 0) {
break;
}
if (!writer_transfer_data(writer, data_buf, (int)read_len)) {
fprintf(stderr, "Failed to write data to Amazon S3\n");
ret = false;
break;
}
} while (read_len == data_len && !S3QueryIsAbortInProgress());
if (ferror(fd)) {
ret = false;
}
fclose(fd);
}
writer_cleanup(&writer);
thread_cleanup();
return ret;
}
// returning uint64_t(-1) means error
uint64_t ChunkBuffer::fill() {
UniqueLock statusLock(&this->statusMutex);
while (this->status != ReadyToFill) {
pthread_cond_wait(&this->statusCondVar, &this->statusMutex);
}
if (S3QueryIsAbortInProgress() || this->isError()) {
this->setSharedError(true);
this->status = ReadyToRead;
pthread_cond_signal(&this->statusCondVar);
return -1;
}
uint64_t offset = this->curFileOffset;
uint64_t leftLen = this->chunkDataSize;
uint64_t readLen = 0;
if (leftLen != 0) {
try {
readLen = this->s3Interface->fetchData(offset, this->chunkData, leftLen, this->s3Url);
if (readLen != leftLen) {
S3DEBUG("Failed to fetch expected data from S3");
this->setSharedError(true, S3PartialResponseError(leftLen, readLen));
} else {
S3DEBUG("Got %" PRIu64 " bytes from S3", readLen);
}
} catch (S3Exception& e) {
S3DEBUG("Failed to fetch expected data from S3");
this->setSharedError(true);
}
}
if (offset + leftLen >= offsetMgr.getKeySize()) {
readLen = 0; // Nothing to read, EOF
S3DEBUG("Reached the end of file");
this->eof = true;
}
this->status = ReadyToRead;
pthread_cond_signal(&this->statusCondVar);
return (this->isError()) ? -1 : readLen;
}
// ret < len means EMPTY
// that's why it checks if leftLen is larger than *or equal to* len below[1], provides a chance ret
// is 0, which is smaller than len. Otherwise, other functions won't know when to read next buffer.
uint64_t ChunkBuffer::read(char* buf, uint64_t len) {
// GPDB abort signal stops s3_import(), this check is not needed if s3_import() every time calls
// ChunkBuffer->Read() only once, otherwise(as we did in downstreamReader->read() for
// decompression feature before), first call sets buffer to ReadyToFill, second call hangs.
S3_CHECK_OR_DIE(!S3QueryIsAbortInProgress(), S3QueryAbort, "");
UniqueLock statusLock(&this->statusMutex);
while (this->status != ReadyToRead) {
pthread_cond_wait(&this->statusCondVar, &this->statusMutex);
}
// Error is shared between all chunks.
if (this->isError()) {
return 0;
}
uint64_t leftLen = this->chunkDataSize - this->curChunkOffset;
uint64_t lenToRead = std::min(len, leftLen);
if (lenToRead != 0) {
memcpy(buf, this->chunkData.data() + this->curChunkOffset, lenToRead);
}
if (len <= leftLen) { // [1]
this->curChunkOffset += lenToRead; // not empty
} else { // empty, reset everything
this->curChunkOffset = 0;
if (!this->isEOF()) {
// Release chunkData memory to reduce consumption.
this->chunkData.release();
this->status = ReadyToFill;
Range range = this->offsetMgr.getNextOffset();
this->curFileOffset = range.offset;
this->chunkDataSize = range.length;
pthread_cond_signal(&this->statusCondVar);
}
}
return lenToRead;
}
// curl's reading function callback.
size_t RESTfulServiceReadFuncCallback(char *ptr, size_t size, size_t nmemb, void *userp) {
if (S3QueryIsAbortInProgress()) {
return CURL_READFUNC_ABORT;
}
UploadData *data = (UploadData *)userp;
uint64_t dataLeft = data->buffer.size() - data->currentPosition;
size_t requestedSize = size * nmemb;
size_t copiedItemNum = requestedSize < dataLeft ? nmemb : (dataLeft / size);
size_t copiedDataSize = copiedItemNum * size;
if (copiedDataSize == 0) return 0;
memcpy(ptr, data->buffer.data() + data->currentPosition, copiedDataSize);
data->currentPosition += copiedDataSize;
return copiedItemNum;
}
void* DownloadThreadFunc(void* data) {
MaskThreadSignals();
ChunkBuffer* buffer = static_cast<ChunkBuffer*>(data);
uint64_t filledSize = 0;
S3DEBUG("Downloading thread starts");
do {
if (S3QueryIsAbortInProgress()) {
S3INFO("Downloading thread is interrupted");
// error is shared between all chunks, so all chunks will stop.
buffer->setSharedError(true, S3QueryAbort("Downloading thread is interrupted"));
// have to unlock ChunkBuffer::read in some certain conditions, for instance, status is
// not ReadyToRead, and read() is waiting for signal stat_cond.
buffer->setStatus(ReadyToRead);
pthread_cond_signal(buffer->getStatCond());
return NULL;
}
filledSize = buffer->fill();
if (filledSize != 0) {
if (buffer->isError()) {
S3DEBUG("Failed to fill downloading buffer");
break;
} else {
S3DEBUG("Size of filled data is %" PRIu64, filledSize);
}
}
} while (!buffer->isEOF());
S3DEBUG("Downloading thread ended");
return NULL;
}
