int __s3fs_remove_object(const char *bucketName, const char *key) { S3_init(); S3BucketContext bucketContext = { 0, bucketName, protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG }; S3ResponseHandler responseHandler = { 0, &responseCompleteCallback }; do { S3_delete_object(&bucketContext, key, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); int result = statusG == S3StatusOK ? 0 : -1; if ((statusG != S3StatusOK) && (statusG != S3StatusErrorPreconditionFailed)) { printError(); } S3_deinitialize(); return result; }
int main(int argc, char** argv) { if (2 > argc){ usage(); return 1; } char *slash = argv[1]; while (*slash && (*slash != '/')) { slash++; } *slash++ = 0; const char *bucketName = argv[1]; const char *key = slash; S3Protocol protocolG = S3ProtocolHTTP; const char* accessKeyIdG = getenv("S3_ACCESS_KEY_ID"); const char* secretAccessKeyG = getenv("S3_SECRET_ACCESS_KEY"); S3Status status; const char* hostname = getenv("S3_HOSTNAME"); if ((status = S3_initialize("s3", S3_INIT_ALL, hostname)) != S3StatusOK) { fprintf(stdout, "Failed to initialize libs3: %s\n", S3_get_status_name(status)); exit(-1); } S3BucketContext bucketContext = { 0, bucketName, protocolG, S3UriStylePath, accessKeyIdG, secretAccessKeyG, 0 }; S3ResponseHandler responseHandler = { 0, &responseCompleteCallback }; //----------------------create bucket-----------------// S3_delete_object(&bucketContext, key, 0, &responseHandler, 0); S3_deinitialize(); return 0; }
S3Status cloud_delete_object(const char *bucketName, const char *key) { S3BucketContext bucketContext = { 0, bucketName, protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG }; S3ResponseHandler responseHandler = { 0, &responseCompleteCallback }; S3_delete_object(&bucketContext, key, 0, &responseHandler, 0); return statusG; }
int s3FileUnlink (rsComm_t *rsComm, char *s3ObjName) { int status; char key[MAX_NAME_LEN], myBucket[MAX_NAME_LEN]; callback_data_t data; S3BucketContext bucketContext; bzero (&data, sizeof (data)); if ((status = parseS3Path (s3ObjName, myBucket, key)) < 0) return status; if ((status = myS3Init ()) != S3StatusOK) return (status); /* {myBucket, 1, 0, S3Auth.accessKeyId, S3Auth.secretAccessKey}; */ /* XXXXX using S3UriStyleVirtualHost causes operation containing * the sub-string "S3" to fail. use S3UriStylePath instead */ /* this initialization failed for 3-2.0 because of the hostName * element S3BucketContext bucketContext = {myBucket, S3ProtocolHTTPS, S3UriStylePath, S3Auth.accessKeyId, S3Auth.secretAccessKey}; */ S3ResponseHandler responseHandler = { 0, &responseCompleteCallback }; S3_delete_object(&bucketContext, key, 0, &responseHandler, &data); if (data.status != S3StatusOK) { status = myS3Error (data.status, S3_FILE_UNLINK_ERR); } else { status = 0; } /* S3_deinitialize(); */ return (status); }
static int ct_archive_data(struct hsm_copyaction_private *hcp, const char *src, const char *dst, int src_fd, const struct hsm_action_item *hai, long hal_flags) { struct hsm_extent he; __u64 file_offset = hai->hai_extent.offset; struct stat src_st; char *uncompress_buf = NULL; char *compress_buf = NULL; __u64 write_total = 0; __u64 length = hai->hai_extent.length; time_t last_report_time; int rc = 0; double start_ct_now = ct_now(); time_t now; int compression_bound = LZ4_compressBound(chunk_size); // Archiving a file from Lustre to the object store CT_TRACE("Archiving %s to %s", src, dst); if (fstat(src_fd, &src_st) < 0) { rc = -errno; CT_ERROR(rc, "cannot stat '%s'", src); return rc; } if (!S_ISREG(src_st.st_mode)) { rc = -EINVAL; CT_ERROR(rc, "'%s' is not a regular file", src); return rc; } if (hai->hai_extent.offset > (__u64)src_st.st_size) { rc = -EINVAL; CT_ERROR(rc, "Trying to start reading past end ("LPU64" > " "%jd) of '%s' source file", hai->hai_extent.offset, (intmax_t)src_st.st_size, src); return rc; } strippingInfo stripping_params; stripping_params.lmm_stripe_count = 1; stripping_params.lmm_stripe_size = ONE_MB; if (ct_save_stripe(src_fd, src, &stripping_params)) { return -1; } /* Don't read beyond a given extent */ if (length > src_st.st_size - hai->hai_extent.offset) length = src_st.st_size - hai->hai_extent.offset; last_report_time = time(NULL); he.offset = file_offset; he.length = 0; rc = llapi_hsm_action_progress(hcp, &he, length, 0); if (rc < 0) { /* Action has been canceled or something wrong * is happening. Stop copying data. */ CT_ERROR(rc, "progress ioctl for copy '%s'->'%s' failed", src, dst); goto out; } errno = 0; uncompress_buf = malloc(chunk_size); if (uncompress_buf == NULL) { rc = -ENOMEM; goto out; } compress_buf = malloc(compression_bound); if (compress_buf == NULL) { rc = -ENOMEM; goto out; } int chunk_id = -1; const char totalLength[] = "totallength"; const char chunksize[] = "chunksize"; const char stripe_size[] = "stripesize"; const char stripe_count[] = "stripecount"; const char path[] = "path"; const char uid[] = "uid"; const char gid[] = "gid"; char totalLength_s[TOTALLENGTH]; char chunksize_s[TOTALLENGTH]; char stripe_size_s[TOTALLENGTH]; char stripe_count_s[TOTALLENGTH]; char path_s[PATH_MAX]; char uid_s[TOTALLENGTH]; char gid_s[TOTALLENGTH]; snprintf(totalLength_s, sizeof(totalLength_s), "%llu", length); snprintf(chunksize_s, sizeof(chunksize_s), "%i", chunk_size); snprintf(stripe_size_s, sizeof(stripe_size_s), "%i", stripping_params.lmm_stripe_size); snprintf(stripe_count_s, sizeof(stripe_count_s), "%i", stripping_params.lmm_stripe_count); snprintf(path_s, sizeof(path_s), "%s", src); // FIXME should use fid2path to get the normal path snprintf(uid_s, sizeof(uid_s), "%i", src_st.st_uid); snprintf(gid_s, sizeof(gid_s), "%i", src_st.st_gid); // Saving some metadata for disaster recovery S3NameValue metadata[7] = { { totalLength, totalLength_s, }, { chunksize, chunksize_s, }, { stripe_size, stripe_size_s, }, { stripe_count, stripe_count_s, }, { path, path_s }, { uid, uid_s }, { gid, gid_s } }; S3PutProperties putProperties = { // application/x-lz4 does not officially exist "application/x-lz4", // contentType NULL, // md5 NULL, // cacheControl NULL, // contentDispositionFilename NULL, // contentEncoding -1, // expires 0, // cannedAcl sizeof(metadata) / sizeof(S3NameValue), // metaDataCount metadata, // S3NameValue *metaData 0, // useServerSideEncryption }; do { // Uploading to object store if (chunk_id == -1) { CT_TRACE("start copy of "LPU64" bytes from '%s' to '%s'", length, src, dst); } // size of the current chunk, limited by chunk_size long long unsigned int chunk; if (length - write_total > chunk_size) { // upper bound is the chunk_size chunk = chunk_size; } else { // limited by the file chunk = length - write_total; } chunk_id = file_offset / chunk_size; put_object_callback_data data; data.buffer_offset = 0; double before_lustre_read = ct_now(); pread(src_fd, uncompress_buf, chunk, file_offset); CT_TRACE("Reading a chunk from %s of %llu bytes offset %llu from lustre took %fs", src, chunk, file_offset, ct_now() - before_lustre_read); double before_compression = ct_now(); int compressed_size = LZ4_compress_default(uncompress_buf, compress_buf, chunk, compression_bound); CT_TRACE("Compressing a chunk from %s took %fs and the compressed size is %i bytes", src, ct_now() - before_compression, compressed_size); if (compressed_size <= 0) { rc = -1; CT_ERROR(rc, "Compression error"); goto out; } data.contentLength = compressed_size; data.buffer = compress_buf; S3PutObjectHandler putObjectHandler = { putResponseHandler, &putObjectDataCallback }; char dst_chunk_s[S3_MAX_KEY_SIZE]; snprintf(dst_chunk_s, sizeof(dst_chunk_s), "%s.%i", dst, chunk_id); char bucket_name[S3_MAX_BUCKET_NAME_SIZE]; getBucketName(sizeof(bucket_name), bucket_name, dst_chunk_s); // Get a local copy of the general bucketContext than overwrite the // pointer to the bucket_name S3BucketContext localbucketContext; memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext)); localbucketContext.bucketName = bucket_name; double before_s3_put = ct_now(); int retry_count = RETRYCOUNT; do { S3_put_object(&localbucketContext, dst_chunk_s, compressed_size, &putProperties, NULL, &putObjectHandler, &data); } while (S3_status_is_retryable(data.status) && should_retry(&retry_count)); CT_TRACE("S3 put of %s took %fs", dst_chunk_s, ct_now() - before_s3_put); if (data.status != S3StatusOK) { rc = -EIO; CT_ERROR(rc, "S3Error %s", S3_get_status_name(data.status)); goto out; } he.offset = file_offset; he.length = chunk; now = time(NULL); if (now >= last_report_time + ct_opt.o_report_int) { last_report_time = now; CT_TRACE("sending progress report for archiving %s", src); rc = llapi_hsm_action_progress(hcp, &he, length, 0); if (rc < 0) { /* Action has been canceled or something wrong * is happening. Stop copying data. */ CT_ERROR(rc, "progress ioctl for copy '%s'->'%s' failed", src, dst); goto out; } } write_total += chunk; file_offset += chunk; } while (file_offset < length); rc = 0; // We need to delete every chunk of higher chunk_id if they // exists, this can happen if the new file is smaller // TODO only delete objects if this is a dirty write chunk_id += 1; do { char dst_s[S3_MAX_KEY_SIZE]; int retry_count; snprintf(dst_s, sizeof(dst_s), "%s.%i", dst, chunk_id); get_object_callback_data head_data; get_object_callback_data delete_data; char bucket_name[S3_MAX_BUCKET_NAME_SIZE]; getBucketName(sizeof(bucket_name), bucket_name, dst_s); // Get a local copy of the general bucketContext than overwrite the // pointer to the bucket_name S3BucketContext localbucketContext; memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext)); localbucketContext.bucketName = bucket_name; CT_TRACE("Checking if chunk %i exists", chunk_id); retry_count = RETRYCOUNT; do { S3_head_object(&localbucketContext, dst_s, NULL, &headResponseHandler, &head_data); } while (S3_status_is_retryable(head_data.status) && should_retry(&retry_count)); if (head_data.status == S3StatusHttpErrorNotFound) { // Object do not exist, this mean we stop deleting chunks CT_TRACE("Chunk %i do not exists", chunk_id); break; } if (head_data.status != S3StatusOK) { rc = -EIO; CT_ERROR(rc, "S3Error %s", S3_get_status_name(head_data.status)); goto out; } CT_TRACE("Deleting chunk %i", chunk_id); retry_count = RETRYCOUNT; do { S3_delete_object(&localbucketContext, dst_s, NULL, &deleteResponseHandler, &delete_data); } while (S3_status_is_retryable(delete_data.status) && should_retry(&retry_count)); if (delete_data.status != S3StatusOK) { rc = -EIO; CT_ERROR(rc, "S3Error %s", S3_get_status_name(delete_data.status)); goto out; } chunk_id++; } while (true); out: if (uncompress_buf != NULL) free(uncompress_buf); if (compress_buf != NULL) free(compress_buf); CT_TRACE("copied "LPU64" bytes in %f seconds", length, ct_now() - start_ct_now); return rc; }
int ct_remove(const struct hsm_action_item *hai, const long hal_flags) { struct hsm_copyaction_private *hcp = NULL; char dst[PATH_MAX]; int rc; int retry_count; char dst_s[S3_MAX_KEY_SIZE]; rc = ct_begin(&hcp, hai); if (rc < 0) goto end_ct_remove; ct_path_archive(dst, sizeof(dst), &hai->hai_fid); CT_TRACE("removing file '%s'", dst); if (ct_opt.o_dry_run) { rc = 0; goto end_ct_remove; } // Get the metadata from the first object to get the number of chunks get_object_callback_data data; snprintf(dst_s, sizeof(dst_s), "%s.0", dst); char bucket_name[S3_MAX_BUCKET_NAME_SIZE]; getBucketName(sizeof(bucket_name), bucket_name, dst_s); // Get a local copy of the general bucketContext than overwrite the // pointer to the bucket_name S3BucketContext localbucketContext; memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext)); localbucketContext.bucketName = bucket_name; retry_count = RETRYCOUNT; do { S3_head_object(&localbucketContext, dst_s, NULL, &headResponseHandler, &data); } while (S3_status_is_retryable(data.status) && should_retry(&retry_count)); if (data.status != S3StatusOK) { rc = -EIO; CT_ERROR(rc, "S3Error %s", S3_get_status_name(data.status)); goto end_ct_remove; } int chunk; for (chunk = data.totalLength / data.chunk_size; chunk >= 0; chunk--) { snprintf(dst_s, sizeof(dst_s), "%s.%i", dst, chunk); get_object_callback_data delete_data; CT_TRACE("Deleting chunk '%s'", dst_s); char bucket_name[S3_MAX_BUCKET_NAME_SIZE]; getBucketName(sizeof(bucket_name), bucket_name, dst_s); // Get a local copy of the general bucketContext than overwrite the // pointer to the bucket_name S3BucketContext localbucketContext; memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext)); localbucketContext.bucketName = bucket_name; retry_count = RETRYCOUNT; do { S3_delete_object(&localbucketContext, dst_s, NULL, &deleteResponseHandler, &delete_data); } while (S3_status_is_retryable(delete_data.status) && should_retry(&retry_count)); if (delete_data.status != S3StatusOK) { rc = -EIO; CT_ERROR(rc, "S3Error %s", S3_get_status_name(delete_data.status)); goto end_ct_remove; } } rc = 0; end_ct_remove: rc = ct_action_done(&hcp, hai, 0, rc); return rc; }