void *DownloadThreadfunc(void *data) {
BlockingBuffer *buffer = reinterpret_cast<BlockingBuffer *>(data);
uint64_t filled_size = 0;
S3INFO("Downloading thread starts");
do {
if (QueryCancelPending) {
S3INFO("Downloading thread is interrupted by GPDB");
return NULL;
}
filled_size = buffer->Fill();
if (filled_size == (uint64_t)-1) {
S3DEBUG("Failed to fill downloading buffer");
} else {
S3DEBUG("Size of filled data is %llu", filled_size);
}
if (buffer->EndOfFile()) break;
if (filled_size == (uint64_t)-1) { // Error
if (buffer->Error()) {
break;
} else {
continue;
}
}
} while (1);
S3INFO("Downloading thread ended");
return NULL;
}
bool Downloader::get(char *data, uint64_t &len) {
uint64_t filelen = this->o->Size();
RETRY:
if (this->readlen == filelen) {
len = 0;
return true;
}
BlockingBuffer *buf = buffers[this->chunkcount % this->num];
uint64_t tmplen = buf->Read(data, len);
this->readlen += tmplen;
if (tmplen < len) {
this->chunkcount++;
if (buf->Error()) {
S3ERROR("Error occurs while downloading, skip");
return false;
}
}
// retry to confirm whether thread reading is finished or chunk size is
// divisible by get()'s buffer size
if (tmplen == 0) {
goto RETRY;
}
len = tmplen;
// S3DEBUG("Got %llu, %llu / %llu", len, this->readlen, filelen);
return true;
}
bool Downloader::plain_get(char *data, uint64_t &len) {
uint64_t filelen = this->o->Size();
uint64_t tmplen = 0;
RETRY:
// confirm there is no more available data, done with this file
if (this->readlen == filelen) {
len = 0;
return true;
}
BlockingBuffer *buf = buffers[this->chunkcount % this->num];
// get data from this->magic_bytes, or buf->Read(), or both
if (this->readlen < this->magic_bytes_num) {
if ((this->readlen + len) <= this->magic_bytes_num) {
memcpy(data, this->magic_bytes + this->readlen, len);
tmplen = len;
} else {
memcpy(data, this->magic_bytes + this->readlen,
this->magic_bytes_num - this->readlen);
tmplen = this->magic_bytes_num - this->readlen +
buf->Read(data + this->magic_bytes_num - this->readlen,
this->readlen + len - this->magic_bytes_num);
}
} else {
tmplen = buf->Read(data, len);
}
this->readlen += tmplen;
if (tmplen < len) {
this->chunkcount++;
if (buf->Error()) {
S3ERROR("Error occurs while downloading, skip");
return false;
}
}
// retry to confirm whether thread reading is finished or chunk size is
// divisible by get()'s buffer size
if (tmplen == 0) {
goto RETRY;
}
len = tmplen;
// S3DEBUG("Got %llu, %llu / %llu", len, this->readlen, filelen);
return true;
}
bool Downloader::get(char *data, uint64_t &len) {
if (this->magic_bytes_num == 0) {
// get first 4(at least 2) bytes to check if this file is compressed
BlockingBuffer *buf = buffers[this->chunkcount % this->num];
if ((this->magic_bytes_num = buf->Read(
(char *)this->magic_bytes, sizeof(this->magic_bytes))) > 1) {
if (!this->set_compression()) {
return false;
}
}
}
switch (this->compression) {
case S3_ZIP_GZIP:
return this->zstream_get(data, len);
break;
default:
return this->plain_get(data, len);
}
}
bool Downloader::zstream_get(char *data, uint64_t &len) {
uint64_t filelen = this->o->Size();
// S3_ZIP_CHUNKSIZE is simply the buffer size for feeding data to and
// pulling data from the zlib routines. 256K is recommended by zlib.
#define S3_ZIP_CHUNKSIZE 256 * 1024
uint32_t left_out = 0;
zstream_info *zinfo = this->z_info;
z_stream *strm = &zinfo->zstream;
RETRY:
// fail-safe, incase(very unlikely) there is a infinite-loop bug. For
// instance, S3 returns wrong file size which is larger than actual the
// number. Never happened, but better be careful.
if (this->chunkcount > (this->o->Size() / this->o->Chunksize() + 2)) {
if (zinfo->inited) {
inflateEnd(strm);
}
len = 0;
return false;
}
// no more available data to read, decompress or copy, done with this file
if (this->readlen == filelen && !(zinfo->have_out - zinfo->done_out) &&
!strm->avail_in) {
if (zinfo->inited) {
inflateEnd(strm);
}
len = 0;
return true;
}
BlockingBuffer *buf = buffers[this->chunkcount % this->num];
// strm is the structure used by zlib to decompress stream
if (!zinfo->inited) {
strm->zalloc = Z_NULL;
strm->zfree = Z_NULL;
strm->opaque = Z_NULL;
strm->avail_in = 0;
strm->next_in = Z_NULL;
zinfo->in = (unsigned char *)malloc(S3_ZIP_CHUNKSIZE);
zinfo->out = (unsigned char *)malloc(S3_ZIP_CHUNKSIZE);
if (!zinfo->in || !zinfo->out) {
S3ERROR("Failed to allocate memory");
return false;
}
// 47 is the number of windows bits, to make sure zlib could recognize
// and decode gzip stream
if (inflateInit2(strm, 47) != Z_OK) {
S3ERROR("Failed to init gzip function");
return false;
}
zinfo->inited = true;
}
do {
// copy decompressed data
left_out = zinfo->have_out - zinfo->done_out;
if (left_out > len) {
memcpy(data, zinfo->out + zinfo->done_out, len);
zinfo->done_out += len;
break;
} else if (left_out) {
memcpy(data, zinfo->out + zinfo->done_out, left_out);
zinfo->done_out = 0;
zinfo->have_out = 0;
len = left_out;
break;
}
// get another decompressed chunk
if (this->readlen && (strm->avail_in != 0)) {
strm->avail_out = S3_ZIP_CHUNKSIZE;
strm->next_out = zinfo->out;
switch (inflate(strm, Z_NO_FLUSH)) {
case Z_STREAM_ERROR:
case Z_NEED_DICT:
case Z_DATA_ERROR:
case Z_MEM_ERROR:
S3ERROR("Failed to decompress data");
inflateEnd(strm);
return false;
}
zinfo->have_out = S3_ZIP_CHUNKSIZE - strm->avail_out;
}
// get another compressed chunk
// from magic_bytes, or buf->Read(), or both
if (!zinfo->have_out) {
if (this->readlen < this->magic_bytes_num) {
memcpy(zinfo->in, this->magic_bytes + this->readlen,
this->magic_bytes_num - this->readlen);
strm->avail_in =
this->magic_bytes_num - this->readlen +
buf->Read((char *)zinfo->in + this->magic_bytes_num -
this->readlen,
S3_ZIP_CHUNKSIZE - this->magic_bytes_num +
this->readlen);
} else {
strm->avail_in = buf->Read((char *)zinfo->in, S3_ZIP_CHUNKSIZE);
}
if (buf->Error()) {
S3ERROR("Error occurs while downloading, skip");
inflateEnd(strm);
return false;
}
strm->next_in = zinfo->in;
// readlen is the read size of orig file, not the decompressed
this->readlen += strm->avail_in;
// done with *reading* this compressed file, still need to confirm
// it's all decompressed and transferred/get()
if (strm->avail_in == 0) {
this->chunkcount++;
goto RETRY;
}
}
} while (1);
return true;
}