int putObject ( char * name, IOBuf * bf )
{
int i;
for ( i = 0 ; i < 9000 ; i ++ )
{
char S[128];
snprintf ( S,sizeof(S), "Ln %d \n" , i );
aws_iobuf_append ( bf,S, strlen(S));
}
return s3_put ( bf, name );
}
int
put_file( IOBuf * aws_buf, char *name ) {
FILE * fp;
char readbuf[BUF_SIZE];
char * filename;
// Read in file to upload
if( (fp = fopen(name, "rb")) == NULL) {
fprintf(stdout, "ERROR: The specified file doesn't exist. \n");
return -1;
}
else {
int n;
while( !feof(fp) ) {
n = fread(readbuf, sizeof(unsigned char), BUF_SIZE, fp);
if(n != BUF_SIZE) {
if(feof(fp)) {
; // Do Nothing
}
else {
fprintf(stdout, "ERROR: Error reading from file. \n");
return -1;
}
}
aws_iobuf_append ( aws_buf, readbuf, n);
}
fclose(fp);
}
// Strip the full path from the name and replace spaces with %20
filename = getFilenameFromPath(name);
int rv = s3_put(aws_buf, filename);
free(filename);
return rv;
}
/// Handles reception of the data
/// \param ptr pointer to the incoming data
/// \param size size of the data member
/// \param nmemb number of data memebers
/// \param stream pointer to I/O buffer
/// \return number of bytes processed
static size_t writefunc ( void * ptr, size_t size, size_t nmemb, void * stream )
{
__debug ( "DATA RCVD %d items of size %d ", nmemb, size );
aws_iobuf_append ( stream, ptr, nmemb*size );
return nmemb * size;
}
size_t streaming_writefunc(void* ptr, size_t size, size_t nmemb, void* stream) {
IOBuf* b = (IOBuf*)stream;
ObjectStream* os = (ObjectStream*)b->user_data;
size_t total = (size * nmemb);
LOG(LOG_INFO, "curl-buff %ld\n", total);
// wait for user-buffer, supplied to stream_get()
WAIT(&os->iob_empty);
LOG(LOG_INFO, "user-buff %ld\n", (b->len - b->write_count));
// maybe we were requested to quit?
if (b->first == NULL) {
// called by stream_sync()
LOG(LOG_INFO, "got QUIT\n");
POST(&os->iob_full);
return 0; // op-thread will fail with CURLE_WRITE_ERROR (?)
}
// check for EOF on the object
if (! total) {
os->flags |= OSF_EOF;
POST(&os->iob_full);
LOG(LOG_INFO, "EOF done\n");
return 0;
}
size_t avail = total; /* availble for reading from curl-buffer */
char* dst = ptr;
size_t writable = (b->len - b->write_count); /* space for writing in user-buffer */
while (avail) {
LOG(LOG_INFO, "iterating: writable=%ld, readble=%ld\n", writable, avail);
// if user-buffer is full, wait for another one
if (! writable) {
LOG(LOG_INFO, "user-buff is full\n");
os->flags |= OSF_EOB; // need fresh buffer from stream_get()
POST(&os->iob_full);
WAIT(&os->iob_empty);
// maybe we were requested to quit?
if (b->first == NULL) {
// called by stream_sync()
LOG(LOG_INFO, "got QUIT\n");
POST(&os->iob_full);
return 0; // op-thread will fail with CURLE_WRITE_ERROR (?)
}
writable = (b->len - b->write_count);
continue;
}
// move data to user's buffer
size_t move = ((writable < avail) ? writable : avail);
aws_iobuf_append(b, dst, move);
avail -= move;
writable -= move;
dst += move;
}
// curl-buffer is exhausted.
LOG(LOG_INFO, "copied all of curl-buff (writable=%ld)\n", writable);
if (writable)
POST(&os->iob_empty); // next curl-callback is pre-approved
else {
os->flags |= OSF_EOB; // need fresh buffer from stream_get()
POST(&os->iob_full);
}
return total; /* to curl */
}
static
IOR_offset_t
S3_Xfer_internal(int access,
void* file,
IOR_size_t* buffer,
IOR_offset_t length,
IOR_param_t* param,
int multi_part_upload_p ) {
if (param->verbose >= VERBOSE_2) {
printf("-> S3_Xfer(acc:%d, target:%s, buf:0x%llx, len:%llu, 0x%llx)\n",
access, (char*)file, buffer, length, param);
}
char* fname = (char*)file; /* see NOTE above S3_Create_Or_Open() */
size_t remaining = (size_t)length;
char* data_ptr = (char *)buffer;
off_t offset = param->offset;
// easier to think
int n_to_n = param->filePerProc;
int n_to_1 = (! n_to_n);
int segmented = (param->segmentCount == 1);
if (access == WRITE) { /* WRITE */
if (verbose >= VERBOSE_3) {
fprintf( stdout, "rank %d writing length=%lld to offset %lld\n",
rank,
remaining,
param->offset + length - remaining);
}
if (multi_part_upload_p) {
// For N:1, part-numbers must have a global ordering for the
// components of the final object. param->part_number is
// incremented by 1 per write, on each rank. This lets us use it
// to compute a global part-numbering.
//
// In the N:N case, we only need to increment part-numbers within
// each rank.
//
// In the N:1 case, the global order of part-numbers we're writing
// depends on whether wer're writing strided or segmented, in
// other words, how <offset> and <remaining> are acutally
// positioning the parts being written. [See discussion at
// S3_Close_internal().]
//
// NOTE: 's3curl.pl --debug' shows StringToSign having partNumber
// first, even if I put uploadId first in the URL. Maybe
// that's what the server will do. GetStringToSign() in
// aws4c is not clever about this, so we spoon-feed args in
// the proper order.
size_t part_number;
if (n_to_1) {
if (segmented) { // segmented
size_t parts_per_rank = param->blockSize / param->transferSize;
part_number = (rank * parts_per_rank) + param->part_number;
}
else // strided
part_number = (param->part_number * param->numTasks) + rank;
}
else
part_number = param->part_number;
++ param->part_number;
// if (verbose >= VERBOSE_3) {
// fprintf( stdout, "rank %d of %d writing (%s,%s) part_number %lld\n",
// rank,
// param->numTasks,
// (n_to_1 ? "N:1" : "N:N"),
// (segmented ? "segmented" : "strided"),
// part_number);
// }
snprintf(buff, BUFF_SIZE,
"%s?partNumber=%d&uploadId=%s",
fname, part_number, param->UploadId);
// For performance, we append <data_ptr> directly into the linked list
// of data in param->io_buf. We are "appending" rather than
// "extending", so the added buffer is seen as written data, rather
// than empty storage.
//
// aws4c parses some header-fields automatically for us (into members
// of the IOBuf). After s3_put2(), we can just read the etag from
// param->io_buf->eTag. The server actually returns literal
// quote-marks, at both ends of the string.
aws_iobuf_reset(param->io_buf);
aws_iobuf_append_static(param->io_buf, data_ptr, remaining);
AWS4C_CHECK( s3_put(param->io_buf, buff) );
AWS4C_CHECK_OK( param->io_buf );
// if (verbose >= VERBOSE_3) {
// printf("rank %d: read ETag = '%s'\n", rank, param->io_buf->eTag);
// if (strlen(param->io_buf->eTag) != ETAG_SIZE+2) { /* quotes at both ends */
// fprintf(stderr, "Rank %d: ERROR: expected ETag to be %d hex digits\n",
// rank, ETAG_SIZE);
// exit(1);
// }
// }
if (verbose >= VERBOSE_3) {
fprintf( stdout, "rank %d of %d (%s,%s) offset %lld, part# %lld --> ETag %s\n",
rank,
param->numTasks,
(n_to_1 ? "N:1" : "N:N"),
(segmented ? "segmented" : "strided"),
offset,
part_number,
param->io_buf->eTag); // incl quote-marks at [0] and [len-1]
}
if (strlen(param->io_buf->eTag) != ETAG_SIZE+2) { /* quotes at both ends */
fprintf(stderr, "Rank %d: ERROR: expected ETag to be %d hex digits\n",
rank, ETAG_SIZE);
exit(1);
}
// save the eTag for later
//
// memcpy(etag, param->io_buf->eTag +1, strlen(param->io_buf->eTag) -2);
// etag[ETAG_SIZE] = 0;
aws_iobuf_append(param->etags,
param->io_buf->eTag +1,
strlen(param->io_buf->eTag) -2);
// DEBUGGING
if (verbose >= VERBOSE_4) {
printf("rank %d: part %d = ETag %s\n", rank, part_number, param->io_buf->eTag);
}
// drop ptrs to <data_ptr>, in param->io_buf
aws_iobuf_reset(param->io_buf);
}
else { // use EMC's byte-range write-support, instead of MPU
// NOTE: You must call 's3_enable_EMC_extensions(1)' for
// byte-ranges to work for writes.
if (n_to_n)
s3_set_byte_range(-1,-1); // EMC header "Range: bytes=-1-" means "append"
else
s3_set_byte_range(offset, remaining);
// For performance, we append <data_ptr> directly into the linked list
// of data in param->io_buf. We are "appending" rather than
// "extending", so the added buffer is seen as written data, rather
// than empty storage.
aws_iobuf_reset(param->io_buf);
aws_iobuf_append_static(param->io_buf, data_ptr, remaining);
AWS4C_CHECK ( s3_put(param->io_buf, file) );
AWS4C_CHECK_OK( param->io_buf );
// drop ptrs to <data_ptr>, in param->io_buf
aws_iobuf_reset(param->io_buf);
}
if ( param->fsyncPerWrite == TRUE ) {
WARN("S3 doesn't support 'fsync'" ); /* does it? */
}
}
else { /* READ or CHECK */
if (verbose >= VERBOSE_3) {
fprintf( stdout, "rank %d reading from offset %lld\n",
rank,
param->offset + length - remaining );
}
// read specific byte-range from the object
// [This is included in the "pure" S3 spec.]
s3_set_byte_range(offset, remaining);
// For performance, we append <data_ptr> directly into the linked
// list of data in param->io_buf. In this case (i.e. reading),
// we're "extending" rather than "appending". That means the
// buffer represents empty storage, which will be filled by the
// libcurl writefunction, invoked via aws4c.
aws_iobuf_reset(param->io_buf);
aws_iobuf_extend_static(param->io_buf, data_ptr, remaining);
AWS4C_CHECK( s3_get(param->io_buf, file) );
if (param->io_buf->code != 206) { /* '206 Partial Content' */
snprintf(buff, BUFF_SIZE,
"Unexpected result (%d, '%s')",
param->io_buf->code, param->io_buf->result);
ERR_SIMPLE(buff);
}
// drop refs to <data_ptr>, in param->io_buf
aws_iobuf_reset(param->io_buf);
}
if (param->verbose >= VERBOSE_2) {
printf("<- S3_Xfer\n");
}
return ( length );
}
