int
getFileFromS3 (char *fileName, char *s3ObjName, rodsLong_t fileSize)
{
#if 0
S3Status status;
#else
int status;
#endif
char key[MAX_NAME_LEN], myBucket[MAX_NAME_LEN];
callback_data_t data;
S3BucketContext bucketContext;
bzero (&data, sizeof (data));
if ((status = parseS3Path (s3ObjName, myBucket, key)) < 0) return status;
data.fd = fopen (fileName, "w+");
if (data.fd == NULL) {
status = UNIX_FILE_OPEN_ERR - errno;
rodsLog (LOG_ERROR,
"getFileFromS3: open error for fileName %s, status = %d",
fileName, status);
return status;
}
data.contentLength = data.originalContentLength = fileSize;
if ((status = myS3Init ()) != S3StatusOK) return (status);
/* {myBucket, 1, 0, S3Auth.accessKeyId, S3Auth.secretAccessKey}; */
/* XXXXX using S3UriStyleVirtualHost causes operation containing
* the sub-string "S3" to fail. use S3UriStylePath instead */
/* this initialization failed for 3-2.0 because of the hostName
* element
S3BucketContext bucketContext =
{myBucket, S3ProtocolHTTPS, S3UriStylePath, S3Auth.accessKeyId,
S3Auth.secretAccessKey}; */
bzero (&bucketContext, sizeof (bucketContext));
bucketContext.bucketName = myBucket;
bucketContext.protocol = S3ProtocolHTTPS;
bucketContext.uriStyle = S3UriStylePath;
bucketContext.accessKeyId = S3Auth.accessKeyId;
bucketContext.secretAccessKey = S3Auth.secretAccessKey;
S3GetObjectHandler getObjectHandler = {
{ &responsePropertiesCallback, &responseCompleteCallback },
&getObjectDataCallback
};
S3_get_object (&bucketContext, key, NULL, 0, fileSize, 0,
&getObjectHandler, &data);
if (data.status != S3StatusOK) {
status = myS3Error (data.status, S3_GET_ERROR);
}
/* S3_deinitialize(); */
fclose (data.fd);
return (status);
}
ssize_t __s3fs_get_object(const char *bucketName, const char *key, uint8_t **buf,
ssize_t start_byte, ssize_t byte_count) {
int64_t ifModifiedSince = -1, ifNotModifiedSince = -1;
const char *ifMatch = 0, *ifNotMatch = 0;
uint64_t startByte = start_byte, byteCount = byte_count;
S3_init();
struct get_callback_data get_context;
get_context.buf = NULL;
get_context.bytes_read = 0;
S3BucketContext bucketContext =
{
0,
bucketName,
protocolG,
uriStyleG,
accessKeyIdG,
secretAccessKeyG
};
S3GetConditions getConditions =
{
ifModifiedSince,
ifNotModifiedSince,
ifMatch,
ifNotMatch
};
S3GetObjectHandler getObjectHandler =
{
{ &responsePropertiesCallback, &responseCompleteCallback },
&getObjectDataCallback
};
do {
S3_get_object(&bucketContext, key, &getConditions, startByte,
byteCount, 0, &getObjectHandler, &get_context);
} while (S3_status_is_retryable(statusG) && should_retry());
ssize_t status = get_context.bytes_read;
if (statusG != S3StatusOK) {
status = -1;
if (get_context.buf) {
free (get_context.buf);
}
printError();
} else {
*buf = get_context.buf;
}
S3_deinitialize();
return status;
}
static int get_s3_object(char *objectName,
get_object_callback_data *data,
S3GetObjectHandler *getObjectHandler){
assert(objectName && data && getObjectHandler);
memset(data, 0, sizeof(get_object_callback_data));
char bucket_name[S3_MAX_BUCKET_NAME_SIZE];
getBucketName(sizeof(bucket_name), bucket_name, objectName);
// Get a local copy of the general bucketContext than overwrite the
// pointer to the bucket_name
S3BucketContext localbucketContext;
memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext));
localbucketContext.bucketName = bucket_name;
data->buffer_offset = 0;
data->buffer = NULL;
double before_s3_get = ct_now();
int retry_count = RETRYCOUNT;
do {
S3_get_object(&localbucketContext, objectName, NULL, 0, 0, NULL, getObjectHandler, data);
} while (S3_status_is_retryable(data->status) && should_retry(&retry_count));
CT_TRACE("S3 get of %s took %fs", objectName, ct_now() - before_s3_get);
if (data->buffer == NULL) {
return -ENOMEM;
}
if (data->status != S3StatusOK) {
CT_ERROR(-EIO, "S3Error %s", S3_get_status_name(data->status));
return -EIO;
}
double before_checksum = ct_now();
unsigned char md5[MD5_DIGEST_LENGTH];
char md5_s[MD5_ASCII];
MD5_CTX mdContext;
MD5_Init (&mdContext);
MD5_Update (&mdContext, data->buffer, data->contentLength);
MD5_Final (md5, &mdContext);
int i;
for(i = 0; i < MD5_DIGEST_LENGTH; i++){
sprintf(&md5_s[i*2], "%02x", md5[i]);
}
if(strcmp(md5_s, data->md5) != 0){
CT_ERROR(-EIO, "Bad MD5 checksum for %s, computed %s, expected %s",
objectName, md5_s, data->md5);
return -EIO;
}
CT_TRACE("Checksum of %s took %fs", objectName, ct_now() - before_checksum);
return 0;
}
static void s3store_task(gpointer a, gpointer s)
{
BlueSkyStoreAsync *async = (BlueSkyStoreAsync *)a;
S3Store *store = (S3Store *)s;
async->status = ASYNC_RUNNING;
async->exec_time = bluesky_now_hires();
if (async->op == STORE_OP_GET) {
struct get_info info;
info.buf = g_string_new("");
info.success = 0;
struct S3GetObjectHandler handler;
handler.responseHandler.propertiesCallback = s3store_properties_callback;
handler.responseHandler.completeCallback = s3store_response_callback;
handler.getObjectDataCallback = s3store_get_handler;
S3_get_object(&store->bucket, async->key, NULL,
async->start, async->len, NULL, &handler, &info);
async->range_done = TRUE;
if (info.success) {
async->data = bluesky_string_new_from_gstring(info.buf);
async->result = 0;
} else {
g_string_free(info.buf, TRUE);
}
} else if (async->op == STORE_OP_PUT) {
struct put_info info;
info.success = 0;
info.val = async->data;
info.offset = 0;
struct S3PutObjectHandler handler;
handler.responseHandler.propertiesCallback
= s3store_properties_callback;
handler.responseHandler.completeCallback = s3store_response_callback;
handler.putObjectDataCallback = s3store_put_handler;
S3_put_object(&store->bucket, async->key, async->data->len, NULL, NULL,
&handler, &info);
if (info.success) {
async->result = 0;
} else {
g_warning("Error completing S3 put operation; client must retry!");
}
}
bluesky_store_async_mark_complete(async);
bluesky_store_async_unref(async);
}
static void do_get(const char *key, size_t bytes)
{
struct callback_state state;
struct S3GetObjectHandler handler;
state.bytes_remaining = bytes;
//state.ts = ts;
handler.responseHandler.propertiesCallback = properties_callback;
handler.responseHandler.completeCallback = complete_callback;
handler.getObjectDataCallback = data_callback;
S3_get_object(&bucket, key, NULL, 0, 0, NULL, &handler, &state);
}
S3Status cloud_get_object(const char *bucketName, const char *key,
get_filler_t filler) {
uint64_t startByte = 0, byteCount = 0;
int64_t ifModifiedSince = -1, ifNotModifiedSince = -1;
const char *ifMatch = 0, *ifNotMatch = 0;
S3BucketContext bucketContext =
{
0,
bucketName,
protocolG,
uriStyleG,
accessKeyIdG,
secretAccessKeyG
};
S3GetConditions getConditions =
{
ifModifiedSince,
ifNotModifiedSince,
ifMatch,
ifNotMatch
};
S3GetObjectHandler getObjectHandler =
{
{ &responsePropertiesCallback, &responseCompleteCallback },
&getObjectDataCallback
};
S3_get_object(&bucketContext, key, &getConditions, startByte,
byteCount, 0, &getObjectHandler, filler);
return statusG;
}
