// invoked by s3_export(), need to be exception safe
GPWriter* writer_init(const char* url_with_options, const char* format) {
GPWriter* writer = NULL;
s3extErrorMessage.clear();
try {
if (!url_with_options) {
return NULL;
}
string urlWithOptions(url_with_options);
S3Params params = InitConfig(urlWithOptions);
InitRemoteLog();
// Prepare memory to be used for thread chunk buffer.
PrepareS3MemContext(params);
string extName = params.isAutoCompress() ? string(format) + ".gz" : format;
writer = new GPWriter(params, extName);
if (writer == NULL) {
return NULL;
}
memoryContextHolder = new S3MemoryContext(params.getMemoryContext());
writer->open(params);
return writer;
} catch (S3Exception& e) {
if (writer != NULL) {
delete writer;
delete memoryContextHolder;
}
s3extErrorMessage =
"writer_init caught a " + e.getType() + " exception: " + e.getFullMessage();
S3ERROR("writer_init caught %s: %s", e.getType().c_str(), s3extErrorMessage.c_str());
return NULL;
} catch (...) {
if (writer != NULL) {
delete writer;
delete memoryContextHolder;
}
S3ERROR("Caught an unexpected exception.");
s3extErrorMessage = "Caught an unexpected exception.";
return NULL;
}
}
// invoked by s3_export(), need to be exception safe
bool writer_cleanup(GPWriter** writer) {
bool result = true;
try {
if (*writer) {
(*writer)->close();
delete *writer;
delete memoryContextHolder;
*writer = NULL;
} else {
result = false;
}
} catch (S3Exception& e) {
s3extErrorMessage =
"writer_cleanup caught a " + e.getType() + " exception: " + e.getFullMessage();
S3ERROR("writer_cleanup caught %s: %s", e.getType().c_str(), s3extErrorMessage.c_str());
result = false;
} catch (...) {
S3ERROR("Caught an unexpected exception.");
s3extErrorMessage = "Caught an unexpected exception.";
result = false;
}
return result;
}
// invoked by s3_export(), need to be exception safe
bool writer_transfer_data(GPWriter* writer, char* data_buf, int data_len) {
try {
if (!writer || !data_buf || (data_len <= 0)) {
return false;
}
uint64_t write_len = writer->write(data_buf, data_len);
S3_CHECK_OR_DIE(write_len == (uint64_t)data_len, S3RuntimeError,
"Failed to upload the data completely.");
} catch (S3Exception& e) {
s3extErrorMessage =
"writer_transfer_data caught a " + e.getType() + " exception: " + e.getFullMessage();
S3ERROR("writer_transfer_data caught %s: %s", e.getType().c_str(),
s3extErrorMessage.c_str());
return false;
} catch (...) {
S3ERROR("Caught an unexpected exception.");
s3extErrorMessage = "Caught an unexpected exception.";
return false;
}
return true;
}
void* S3KeyWriter::UploadThreadFunc(void* data) {
MaskThreadSignals();
ThreadParams* params = (ThreadParams*)data;
S3KeyWriter* writer = params->keyWriter;
try {
S3DEBUG("Upload thread start: %" PRIX64 ", part number: %" PRIu64 ", data size: %zu",
(uint64_t) pthread_self(), params->currentNumber, params->data.size());
string etag = writer->s3Interface->uploadPartOfData(
params->data, writer->params.getS3Url(), params->currentNumber, writer->uploadId);
// when unique_lock destructs it will automatically unlock the mutex.
UniqueLock threadLock(&writer->mutex);
// etag is empty if the query is cancelled by user.
if (!etag.empty()) {
writer->etagList[params->currentNumber] = etag;
}
writer->activeThreads--;
pthread_cond_broadcast(&writer->cv);
S3DEBUG("Upload part finish: %" PRIX64 ", eTag: %s, part number: %" PRIu64, (uint64_t) pthread_self(),
etag.c_str(), params->currentNumber);
} catch (S3Exception& e) {
S3ERROR("Upload thread error: %s", e.getMessage().c_str());
UniqueLock exceptLock(&writer->exceptionMutex);
writer->sharedError = true;
writer->sharedException = std::current_exception();
// notify the flushBuffer, otherwise it will be locked when trying to create a new thread.
writer->activeThreads--;
pthread_cond_broadcast(&writer->cv);
}
delete params;
return NULL;
}
bool S3Reader_fake::Init(int segid, int segnum, int chunksize) {
// set segment id and num
this->segid = segid; // fake
this->segnum = segnum; // fake
this->contentindex = this->segid;
this->chunksize = chunksize;
// Validate url first
if (!this->ValidateURL()) {
S3ERROR("validate url fail %s\n", this->url.c_str());
}
// TODO: As separated function for generating url
this->keylist = ListBucket_FakeHTTP("localhost", this->bucket.c_str());
// this->keylist = ListBucket_FakeHTTP("localhost", "metro.pivotal.io");
if (!this->keylist) {
return false;
}
this->getNextDownloader();
return this->filedownloader ? true : false;
}
