void S3KeyWriter::checkQueryCancelSignal() { if (S3QueryIsAbortInProgress() && !this->uploadId.empty()) { // to avoid dead-lock when other upload threads hold the lock pthread_mutex_unlock(&this->mutex); // wait for all threads to complete for (size_t i = 0; i < threadList.size(); i++) { pthread_join(threadList[i], NULL); } this->threadList.clear(); // to avoid double unlock as other parts may lock it pthread_mutex_lock(&this->mutex); S3DEBUG("Start aborting multipart uploading (uploadID: %s, %lu parts uploaded)", this->uploadId.c_str(), this->etagList.size()); this->s3Interface->abortUpload(this->params.getS3Url(), this->uploadId); S3DEBUG("Finished aborting multipart uploading (uploadID: %s)", this->uploadId.c_str()); this->etagList.clear(); this->uploadId.clear(); S3_DIE(S3QueryAbort, "Uploading is interrupted"); } }
void* DownloadThreadFunc(void* data) { ChunkBuffer* buffer = static_cast<ChunkBuffer*>(data); uint64_t filledSize = 0; S3DEBUG("Downloading thread starts"); do { if (QueryCancelPending) { S3INFO("Downloading thread is interrupted by GPDB"); // error is shared between all chunks, so all chunks will stop. buffer->setSharedError(true, "Downloading thread is interrupted by GPDB"); // have to unlock ChunkBuffer::read in some certain conditions, for instance, status is // not ReadyToRead, and read() is waiting for signal stat_cond. buffer->setStatus(ReadyToRead); pthread_cond_signal(buffer->getStatCond()); return NULL; } filledSize = buffer->fill(); if (filledSize != 0) { if (buffer->isError()) { S3DEBUG("Failed to fill downloading buffer"); break; } else { S3DEBUG("Size of filled data is %" PRIu64, filledSize); } } } while (!buffer->isEOF()); S3DEBUG("Downloading thread ended"); return NULL; }
void S3KeyWriter::completeKeyWriting() { // make sure the buffer is clear this->flushBuffer(); // wait for all threads to complete for (size_t i = 0; i < threadList.size(); i++) { pthread_join(threadList[i], NULL); } this->threadList.clear(); this->checkQueryCancelSignal(); vector<string> etags; // it is equivalent to foreach(e in etagList) push_back(e.second); // transform(etagList.begin(), etagList.end(), etags.begin(), // [](std::pair<const uint64_t, string>& p) { return p.second; }); etags.reserve(etagList.size()); for (map<uint64_t, string>::iterator i = etagList.begin(); i != etagList.end(); i++) { etags.push_back(i->second); } if (!this->etagList.empty() && !this->uploadId.empty()) { this->s3Interface->completeMultiPart(this->params.getS3Url(), this->uploadId, etags); } S3DEBUG("Segment %d has finished uploading \"%s\"", s3ext_segid, this->params.getS3Url().getFullUrlForCurl().c_str()); this->buffer.clear(); this->etagList.clear(); this->uploadId.clear(); }
uint64_t S3BucketReader::read(char* buf, uint64_t count) { S3_CHECK_OR_DIE(this->upstreamReader != NULL, S3RuntimeError, "upstreamReader is NULL"); uint64_t readCount = 0; while (true) { if (this->needNewReader) { if (this->keyIndex >= this->keyList.contents.size()) { S3DEBUG("Read finished for segment: %d", s3ext_segid); return 0; } BucketContent& key = this->getNextKey(); this->upstreamReader->open(constructReaderParams(key)); this->needNewReader = false; // ignore header line if it is not the first file if (hasHeader && !this->isFirstFile) { readCount = readWithoutHeaderLine(buf, count); if (readCount != 0) { return readCount; } } } readCount = this->upstreamReader->read(buf, count); if (readCount != 0) { return readCount; } // Finished one file, continue to next this->upstreamReader->close(); this->needNewReader = true; this->isFirstFile = false; } }
// returning uint64_t(-1) means error uint64_t ChunkBuffer::fill() { UniqueLock statusLock(&this->statusMutex); while (this->status != ReadyToFill) { pthread_cond_wait(&this->statusCondVar, &this->statusMutex); } if (S3QueryIsAbortInProgress() || this->isError()) { this->setSharedError(true); this->status = ReadyToRead; pthread_cond_signal(&this->statusCondVar); return -1; } uint64_t offset = this->curFileOffset; uint64_t leftLen = this->chunkDataSize; uint64_t readLen = 0; if (leftLen != 0) { try { readLen = this->s3Interface->fetchData(offset, this->chunkData, leftLen, this->s3Url); if (readLen != leftLen) { S3DEBUG("Failed to fetch expected data from S3"); this->setSharedError(true, S3PartialResponseError(leftLen, readLen)); } else { S3DEBUG("Got %" PRIu64 " bytes from S3", readLen); } } catch (S3Exception& e) { S3DEBUG("Failed to fetch expected data from S3"); this->setSharedError(true); } } if (offset + leftLen >= offsetMgr.getKeySize()) { readLen = 0; // Nothing to read, EOF S3DEBUG("Reached the end of file"); this->eof = true; } this->status = ReadyToRead; pthread_cond_signal(&this->statusCondVar); return (this->isError()) ? -1 : readLen; }
// returning -1 means error uint64_t ChunkBuffer::fill() { pthread_mutex_lock(&this->statusMutex); while (this->status != ReadyToFill) { pthread_cond_wait(&this->statusCondVar, &this->statusMutex); } uint64_t offset = this->curFileOffset; uint64_t leftLen = this->chunkDataSize; uint64_t readLen = 0; if (leftLen != 0) { try { readLen = this->s3interface->fetchData( offset, this->chunkData, leftLen, this->sourceUrl, this->sharedKeyReader.getRegion(), this->sharedKeyReader.getCredential()); } catch (std::exception& e) { S3DEBUG("Failed to fetch expected data from S3"); this->setSharedError(true, e.what()); } if (readLen != leftLen) { S3DEBUG("Failed to fetch expected data from S3"); this->setSharedError(true, "Failed to fetch expected data from S3"); } else { S3DEBUG("Got %" PRIu64 " bytes from S3", readLen); } } if (offset + leftLen >= offsetMgr.getKeySize()) { readLen = 0; // Nothing to read, EOF S3DEBUG("Reached the end of file"); this->eof = true; } this->status = ReadyToRead; pthread_cond_signal(&this->statusCondVar); pthread_mutex_unlock(&this->statusMutex); return (this->isError()) ? -1 : readLen; }
void S3KeyWriter::open(const S3Params& params) { this->params = params; S3_CHECK_OR_DIE(this->s3Interface != NULL, S3RuntimeError, "s3Interface must not be NULL"); S3_CHECK_OR_DIE(this->params.getChunkSize() > 0, S3RuntimeError, "chunkSize must not be zero"); buffer.reserve(this->params.getChunkSize()); this->uploadId = this->s3Interface->getUploadId(this->params.getS3Url()); S3_CHECK_OR_DIE(!this->uploadId.empty(), S3RuntimeError, "Failed to get upload id"); S3DEBUG("key: %s, upload id: %s", this->params.getS3Url().getFullUrlForCurl().c_str(), this->uploadId.c_str()); }
S3Params S3BucketReader::constructReaderParams(BucketContent& key) { // encode the key name but leave the "/" // "/encoded_path/encoded_name" string keyEncoded = UriEncode(key.getName()); FindAndReplace(keyEncoded, "%2F", "/"); S3Params readerParams = this->params.setPrefix(keyEncoded); readerParams.setKeySize(key.getSize()); S3DEBUG("key: %s, size: %" PRIu64, readerParams.getS3Url().getFullUrlForCurl().c_str(), readerParams.getKeySize()); return readerParams; }
void* S3KeyWriter::UploadThreadFunc(void* data) { MaskThreadSignals(); ThreadParams* params = (ThreadParams*)data; S3KeyWriter* writer = params->keyWriter; try { S3DEBUG("Upload thread start: %" PRIX64 ", part number: %" PRIu64 ", data size: %zu", (uint64_t) pthread_self(), params->currentNumber, params->data.size()); string etag = writer->s3Interface->uploadPartOfData( params->data, writer->params.getS3Url(), params->currentNumber, writer->uploadId); // when unique_lock destructs it will automatically unlock the mutex. UniqueLock threadLock(&writer->mutex); // etag is empty if the query is cancelled by user. if (!etag.empty()) { writer->etagList[params->currentNumber] = etag; } writer->activeThreads--; pthread_cond_broadcast(&writer->cv); S3DEBUG("Upload part finish: %" PRIX64 ", eTag: %s, part number: %" PRIu64, (uint64_t) pthread_self(), etag.c_str(), params->currentNumber); } catch (S3Exception& e) { S3ERROR("Upload thread error: %s", e.getMessage().c_str()); UniqueLock exceptLock(&writer->exceptionMutex); writer->sharedError = true; writer->sharedException = std::current_exception(); // notify the flushBuffer, otherwise it will be locked when trying to create a new thread. writer->activeThreads--; pthread_cond_broadcast(&writer->cv); } delete params; return NULL; }
S3Params S3BucketReader::constructReaderParams(BucketContent& key) { S3Params readerParams = this->params; // encode the key name but leave the "/" // "/encoded_path/encoded_name" string keyEncoded = uri_encode(key.getName()); find_replace(keyEncoded, "%2F", "/"); readerParams.setKeyUrl(this->getKeyURL(keyEncoded)); readerParams.setRegion(this->region); readerParams.setKeySize(key.getSize()); S3DEBUG("key: %s, size: %" PRIu64, readerParams.getKeyUrl().c_str(), readerParams.getKeySize()); return readerParams; }
void CompressWriter::close() { if (this->isClosed) { return; } int status; do { status = deflate(&this->zstream, Z_FINISH); this->flush(); } while (status == Z_OK); deflateEnd(&this->zstream); if (status != Z_STREAM_END) { CHECK_OR_DIE_MSG(false, "Failed to finish data compression: %d, %s", status, this->zstream.msg); } S3DEBUG("Compression finished: Z_STREAM_END."); this->writer->close(); this->isClosed = true; }
void CompressWriter::close() { if (this->isClosed) { return; } int status; do { status = deflate(&this->zstream, Z_FINISH); this->flush(); } while (status == Z_OK); deflateEnd(&this->zstream); if (status != Z_STREAM_END) { S3_CHECK_OR_DIE(false, S3RuntimeError, string("Failed to compress data: ") + std::to_string((unsigned long long)status) + ", " + this->zstream.msg); } S3DEBUG("Compression finished: Z_STREAM_END."); this->writer->close(); this->isClosed = true; }