bool downloadS3(const char *urlWithOptions) { if (!urlWithOptions) { return false; } int data_len = BUF_SIZE; char data_buf[BUF_SIZE]; bool ret = true; thread_setup(); GPReader *reader = reader_init(urlWithOptions); if (!reader) { return false; } do { data_len = BUF_SIZE; if (!reader_transfer_data(reader, data_buf, data_len)) { fprintf(stderr, "Failed to read data from Amazon S3\n"); ret = false; break; } fwrite(data_buf, (size_t)data_len, 1, stdout); } while (data_len && !S3QueryIsAbortInProgress()); reader_cleanup(&reader); thread_cleanup(); return ret; }
void S3KeyWriter::checkQueryCancelSignal() { if (S3QueryIsAbortInProgress() && !this->uploadId.empty()) { // to avoid dead-lock when other upload threads hold the lock pthread_mutex_unlock(&this->mutex); // wait for all threads to complete for (size_t i = 0; i < threadList.size(); i++) { pthread_join(threadList[i], NULL); } this->threadList.clear(); // to avoid double unlock as other parts may lock it pthread_mutex_lock(&this->mutex); S3DEBUG("Start aborting multipart uploading (uploadID: %s, %lu parts uploaded)", this->uploadId.c_str(), this->etagList.size()); this->s3Interface->abortUpload(this->params.getS3Url(), this->uploadId); S3DEBUG("Finished aborting multipart uploading (uploadID: %s)", this->uploadId.c_str()); this->etagList.clear(); this->uploadId.clear(); S3_DIE(S3QueryAbort, "Uploading is interrupted"); } }
// curl's headers write function callback. size_t RESTfulServiceHeadersWriteFuncCallback(char *ptr, size_t size, size_t nmemb, void *userp) { if (S3QueryIsAbortInProgress()) { return 0; } size_t realsize = size * nmemb; Response *resp = (Response *)userp; resp->appendHeadersBuffer(ptr, realsize); return realsize; }
int main(int argc, char *argv[]) { bool ret = true; s3ext_loglevel = EXT_ERROR; s3ext_logtype = STDERR_LOG; if (argc == 1) { printUsage(stderr); exit(EXIT_FAILURE); } /* Prepare to receive interrupts */ registerSignalHandler(); map<char, string> optionPairs = parseCommandLineArgs(argc, argv); validateCommandLineArgs(optionPairs); if (!optionPairs.empty()) { const char *arg = optionPairs.begin()->second.c_str(); switch (optionPairs.begin()->first) { case 'c': ret = checkConfig(arg); break; case 'd': ret = downloadS3(arg); break; case 'u': case 'f': ret = uploadS3(optionPairs['u'].c_str(), optionPairs['f'].c_str()); break; case 'h': printUsage(stdout); break; case 't': printTemplate(); break; default: printUsage(stderr); exit(EXIT_FAILURE); } } // Abort should not print the failed info if (ret || S3QueryIsAbortInProgress()) { exit(EXIT_SUCCESS); } else { fprintf(stderr, "Failed. Please check the arguments and configuration file.\n\n"); printUsage(stderr); exit(EXIT_FAILURE); } }
bool uploadS3(const char *urlWithOptions, const char *fileToUpload) { if (!urlWithOptions) { return false; } size_t data_len = BUF_SIZE; char data_buf[BUF_SIZE]; size_t read_len = 0; bool ret = true; thread_setup(); GPWriter *writer = writer_init(urlWithOptions); if (!writer) { return false; } FILE *fd = fopen(fileToUpload, "r"); if (fd == NULL) { fprintf(stderr, "File does not exist\n"); ret = false; } else { do { read_len = fread(data_buf, 1, data_len, fd); if (read_len == 0) { break; } if (!writer_transfer_data(writer, data_buf, (int)read_len)) { fprintf(stderr, "Failed to write data to Amazon S3\n"); ret = false; break; } } while (read_len == data_len && !S3QueryIsAbortInProgress()); if (ferror(fd)) { ret = false; } fclose(fd); } writer_cleanup(&writer); thread_cleanup(); return ret; }
// returning uint64_t(-1) means error uint64_t ChunkBuffer::fill() { UniqueLock statusLock(&this->statusMutex); while (this->status != ReadyToFill) { pthread_cond_wait(&this->statusCondVar, &this->statusMutex); } if (S3QueryIsAbortInProgress() || this->isError()) { this->setSharedError(true); this->status = ReadyToRead; pthread_cond_signal(&this->statusCondVar); return -1; } uint64_t offset = this->curFileOffset; uint64_t leftLen = this->chunkDataSize; uint64_t readLen = 0; if (leftLen != 0) { try { readLen = this->s3Interface->fetchData(offset, this->chunkData, leftLen, this->s3Url); if (readLen != leftLen) { S3DEBUG("Failed to fetch expected data from S3"); this->setSharedError(true, S3PartialResponseError(leftLen, readLen)); } else { S3DEBUG("Got %" PRIu64 " bytes from S3", readLen); } } catch (S3Exception& e) { S3DEBUG("Failed to fetch expected data from S3"); this->setSharedError(true); } } if (offset + leftLen >= offsetMgr.getKeySize()) { readLen = 0; // Nothing to read, EOF S3DEBUG("Reached the end of file"); this->eof = true; } this->status = ReadyToRead; pthread_cond_signal(&this->statusCondVar); return (this->isError()) ? -1 : readLen; }
// ret < len means EMPTY // that's why it checks if leftLen is larger than *or equal to* len below[1], provides a chance ret // is 0, which is smaller than len. Otherwise, other functions won't know when to read next buffer. uint64_t ChunkBuffer::read(char* buf, uint64_t len) { // GPDB abort signal stops s3_import(), this check is not needed if s3_import() every time calls // ChunkBuffer->Read() only once, otherwise(as we did in downstreamReader->read() for // decompression feature before), first call sets buffer to ReadyToFill, second call hangs. S3_CHECK_OR_DIE(!S3QueryIsAbortInProgress(), S3QueryAbort, ""); UniqueLock statusLock(&this->statusMutex); while (this->status != ReadyToRead) { pthread_cond_wait(&this->statusCondVar, &this->statusMutex); } // Error is shared between all chunks. if (this->isError()) { return 0; } uint64_t leftLen = this->chunkDataSize - this->curChunkOffset; uint64_t lenToRead = std::min(len, leftLen); if (lenToRead != 0) { memcpy(buf, this->chunkData.data() + this->curChunkOffset, lenToRead); } if (len <= leftLen) { // [1] this->curChunkOffset += lenToRead; // not empty } else { // empty, reset everything this->curChunkOffset = 0; if (!this->isEOF()) { // Release chunkData memory to reduce consumption. this->chunkData.release(); this->status = ReadyToFill; Range range = this->offsetMgr.getNextOffset(); this->curFileOffset = range.offset; this->chunkDataSize = range.length; pthread_cond_signal(&this->statusCondVar); } } return lenToRead; }
// curl's reading function callback. size_t RESTfulServiceReadFuncCallback(char *ptr, size_t size, size_t nmemb, void *userp) { if (S3QueryIsAbortInProgress()) { return CURL_READFUNC_ABORT; } UploadData *data = (UploadData *)userp; uint64_t dataLeft = data->buffer.size() - data->currentPosition; size_t requestedSize = size * nmemb; size_t copiedItemNum = requestedSize < dataLeft ? nmemb : (dataLeft / size); size_t copiedDataSize = copiedItemNum * size; if (copiedDataSize == 0) return 0; memcpy(ptr, data->buffer.data() + data->currentPosition, copiedDataSize); data->currentPosition += copiedDataSize; return copiedItemNum; }
void* DownloadThreadFunc(void* data) { MaskThreadSignals(); ChunkBuffer* buffer = static_cast<ChunkBuffer*>(data); uint64_t filledSize = 0; S3DEBUG("Downloading thread starts"); do { if (S3QueryIsAbortInProgress()) { S3INFO("Downloading thread is interrupted"); // error is shared between all chunks, so all chunks will stop. buffer->setSharedError(true, S3QueryAbort("Downloading thread is interrupted")); // have to unlock ChunkBuffer::read in some certain conditions, for instance, status is // not ReadyToRead, and read() is waiting for signal stat_cond. buffer->setStatus(ReadyToRead); pthread_cond_signal(buffer->getStatCond()); return NULL; } filledSize = buffer->fill(); if (filledSize != 0) { if (buffer->isError()) { S3DEBUG("Failed to fill downloading buffer"); break; } else { S3DEBUG("Size of filled data is %" PRIu64, filledSize); } } } while (!buffer->isEOF()); S3DEBUG("Downloading thread ended"); return NULL; }