TEST_FIXTURE(append_blob_test_base, append_block_size)
{
const size_t buffer_size = 8 * 1024 * 1024;
std::vector<uint8_t> buffer;
buffer.reserve(buffer_size);
azure::storage::blob_request_options options;
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
size_t sizes[] = { 1, 2, 1023, 1024, 4 * 1024, 1024 * 1024, azure::storage::protocol::max_block_size - 1, azure::storage::protocol::max_block_size };
size_t invalid_sizes[] = { azure::storage::protocol::max_block_size + 1, 6 * 1024 * 1024, 8 * 1024 * 1024 };
int64_t bytes_appended = 0;
options.set_use_transactional_md5(true);
for (size_t size : sizes)
{
buffer.resize(size);
auto md5 = fill_buffer_and_get_md5(buffer, 0, size);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, md5, azure::storage::access_condition(), options, m_context);
CHECK_EQUAL(bytes_appended, offset);
bytes_appended += size;
}
options.set_use_transactional_md5(false);
for (size_t size : invalid_sizes)
{
buffer.resize(size);
fill_buffer_and_get_md5(buffer, 0, size);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition(), options, m_context), azure::storage::storage_exception);
}
}
TEST_FIXTURE(page_blob_test_base, page_blob_write_stream_seek_without_md5)
{
std::vector<uint8_t> buffer;
std::vector<uint8_t> final_blob_contents;
final_blob_contents.resize(16 * 1024);
azure::storage::blob_request_options options;
options.set_store_blob_content_md5(false);
auto stream = m_blob.open_write(final_blob_contents.size(), 0, azure::storage::access_condition(), options, m_context);
CHECK(stream.can_seek());
// Because container create and blob create are also in the request results,
// number of requests should start with 2.
size_t attempts = 2;
buffer.resize(1024);
fill_buffer_and_get_md5(buffer);
stream.streambuf().putn_nocopy(buffer.data(), buffer.size()).wait();
std::copy(buffer.begin(), buffer.end(), final_blob_contents.begin());
stream.seek(5 * 1024);
attempts++;
fill_buffer_and_get_md5(buffer);
stream.streambuf().putn_nocopy(buffer.data(), buffer.size()).wait();
std::copy(buffer.begin(), buffer.end(), final_blob_contents.begin() + 5 * 1024);
fill_buffer_and_get_md5(buffer);
stream.streambuf().putn_nocopy(buffer.data(), buffer.size()).wait();
std::copy(buffer.begin(), buffer.end(), final_blob_contents.begin() + 6 * 1024);
stream.seek(512);
attempts++;
fill_buffer_and_get_md5(buffer);
stream.streambuf().putn_nocopy(buffer.data(), buffer.size()).wait();
std::copy(buffer.begin(), buffer.end(), final_blob_contents.begin() + 512);
stream.close().wait();
attempts++;
CHECK_EQUAL(attempts, m_context.request_results().size());
concurrency::streams::container_buffer<std::vector<uint8_t>> downloaded_blob;
m_blob.download_to_stream(downloaded_blob.create_ostream(), azure::storage::access_condition(), options, m_context);
CHECK_ARRAY_EQUAL(final_blob_contents, downloaded_blob.collection(), (int)final_blob_contents.size());
}
TEST_FIXTURE(page_blob_test_base, existing_page_blob_write_stream)
{
CHECK_THROW(m_blob.open_write(azure::storage::access_condition(), azure::storage::blob_request_options(), m_context), azure::storage::storage_exception);
azure::storage::blob_request_options options;
options.set_store_blob_content_md5(true);
std::vector<uint8_t> buffer;
buffer.resize(16 * 1024);
fill_buffer_and_get_md5(buffer);
m_blob.upload_from_stream(concurrency::streams::bytestream::open_istream(buffer), 0, azure::storage::access_condition(), options, m_context);
CHECK_THROW(m_blob.open_write(azure::storage::access_condition(), options, m_context), std::logic_error);
options.set_store_blob_content_md5(false);
auto stream = m_blob.open_write(azure::storage::access_condition(), options, m_context);
stream.seek(512);
stream.streambuf().putn_nocopy(buffer.data(), 512).wait();
stream.close().wait();
concurrency::streams::container_buffer<std::vector<uint8_t>> downloaded_blob;
CHECK_THROW(m_blob.download_to_stream(downloaded_blob.create_ostream(), azure::storage::access_condition(), options, m_context), azure::storage::storage_exception);
downloaded_blob.seekpos(0, std::ios_base::out);
options.set_disable_content_md5_validation(true);
m_blob.download_to_stream(downloaded_blob.create_ostream(), azure::storage::access_condition(), options, m_context);
CHECK_ARRAY_EQUAL(buffer.data(), downloaded_blob.collection().data(), 512);
CHECK_ARRAY_EQUAL(buffer.data(), downloaded_blob.collection().data() + 512, 512);
CHECK_ARRAY_EQUAL(buffer.data() + 1024, downloaded_blob.collection().data() + 1024, (int)(buffer.size()) - 1024);
}
TEST_FIXTURE(append_blob_test_base, append_block_append_position_condition)
{
const size_t buffer_size = 64 * 1024;
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
azure::storage::blob_request_options options;
options.set_use_transactional_md5(true);
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
int64_t invalid_appendpos[] = { 1, 2, buffer_size, buffer_size + 1, std::numeric_limits<int64_t>::max() };
for (int64_t appendpos : invalid_appendpos)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.append_block(stream, md5, azure::storage::access_condition::generate_if_append_position_equal_condition(appendpos), options, m_context), azure::storage::storage_exception);
}
for (int16_t i = 0; i < 3; i++)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, md5, azure::storage::access_condition::generate_if_append_position_equal_condition(i * buffer_size), options, m_context);
CHECK_EQUAL(offset, i * buffer_size);
CHECK_EQUAL(i + 1, m_blob.properties().append_blob_committed_block_count());
}
int64_t invalid_appendpos2[] = { buffer_size * 3 - 1, buffer_size * 3 + 1};
for (int64_t appendpos : invalid_appendpos2)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.append_block(stream, md5, azure::storage::access_condition::generate_if_append_position_equal_condition(appendpos), options, m_context), azure::storage::storage_exception);
}
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_EQUAL(3 * buffer_size, m_blob.append_block(stream, md5, azure::storage::access_condition::generate_if_append_position_equal_condition(3 * buffer_size), options, m_context));
CHECK_EQUAL(4, m_blob.properties().append_blob_committed_block_count());
m_blob.delete_blob();
}
TEST_FIXTURE(append_blob_test_base, append_blob_upload_max_size_condition)
{
const size_t buffer_size = 1024 * 1024;
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
fill_buffer_and_get_md5(buffer);
concurrency::streams::istream stream = concurrency::streams::bytestream::open_istream(buffer);
auto condition = azure::storage::access_condition::generate_if_max_size_less_than_or_equal_condition(512);
CHECK_THROW(m_blob.upload_from_stream(stream, condition, azure::storage::blob_request_options(), m_context), std::invalid_argument);
}
TEST_FIXTURE(block_blob_test_base, existing_block_blob_stream_seek_read_getn)
{
azure::storage::blob_request_options options;
const size_t buffer_size = 64 * 1024;
std::vector<uint8_t> input_buffer;
input_buffer.resize(buffer_size);
fill_buffer_and_get_md5(input_buffer);
m_blob.upload_from_stream(concurrency::streams::bytestream::open_istream(input_buffer), azure::storage::access_condition(), options, m_context);
auto blob_stream = m_blob.open_read(azure::storage::access_condition(), options, m_context);
seek_and_read_getn(blob_stream, input_buffer);
blob_stream.close().wait();
}
TEST_FIXTURE(page_blob_test_base, existing_page_blob_stream_seek_write_putn)
{
azure::storage::blob_request_options options;
const size_t buffer_size = 64 * 1024;
std::vector<uint8_t> original_buffer;
original_buffer.resize(buffer_size);
fill_buffer_and_get_md5(original_buffer);
m_blob.upload_from_stream(concurrency::streams::bytestream::open_istream(original_buffer), 0, azure::storage::access_condition(), options, m_context);
CHECK_EQUAL(buffer_size, m_blob.properties().size());
auto blob_stream = m_blob.open_write(azure::storage::access_condition(), options, m_context);
seek_and_write_putn(blob_stream, original_buffer);
blob_stream.close().wait();
}
TEST_FIXTURE(block_blob_test_base, blob_read_stream_download)
{
azure::storage::blob_request_options options;
options.set_stream_read_size_in_bytes(1 * 1024 * 1024);
options.set_use_transactional_md5(true);
std::vector<uint8_t> buffer;
buffer.resize(3 * 1024 * 1024);
fill_buffer_and_get_md5(buffer);
m_blob.upload_from_stream(concurrency::streams::bytestream::open_istream(buffer), azure::storage::access_condition(), options, m_context);
concurrency::streams::container_buffer<std::vector<uint8_t>> output_buffer;
auto stream = m_blob.open_read(azure::storage::access_condition(), options, m_context);
stream.read_to_end(output_buffer).wait();
stream.close().wait();
CHECK_EQUAL(buffer.size(), output_buffer.collection().size());
CHECK_ARRAY_EQUAL(buffer, output_buffer.collection(), (int)output_buffer.collection().size());
}
TEST_FIXTURE(block_blob_test_base, blob_read_stream_etag_lock)
{
azure::storage::blob_request_options options;
options.set_stream_read_size_in_bytes(1 * 1024 * 1024);
std::vector<uint8_t> buffer;
buffer.resize(2 * 1024 * 1024);
fill_buffer_and_get_md5(buffer);
m_blob.upload_from_stream(concurrency::streams::bytestream::open_istream(buffer), azure::storage::access_condition(), options, m_context);
auto stream = m_blob.open_read(azure::storage::access_condition(), options, m_context);
m_blob.upload_metadata(azure::storage::access_condition(), options, m_context);
CHECK_THROW(stream.read().wait(), azure::storage::storage_exception);
stream.close().wait();
auto condition = azure::storage::access_condition::generate_if_match_condition(_XPLATSTR("*"));
stream = m_blob.open_read(condition, options, m_context);
m_blob.upload_metadata(azure::storage::access_condition(), options, m_context);
CHECK_THROW(stream.read().wait(), azure::storage::storage_exception);
stream.close().wait();
}
TEST_FIXTURE(block_blob_test_base, storage_extended_error_verify_xml_with_details)
{
const size_t buffer_size = 8 * 1024;
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
m_blob.upload_block(get_block_id(0), stream, md5);
try
{
stream.seek(1024, std::ios_base::beg);
m_blob.upload_block(get_block_id(1), stream, md5);
CHECK(false);
}
catch (azure::storage::storage_exception& ex)
{
CHECK_UTF8_EQUAL(U("Md5Mismatch"), ex.result().extended_error().code());
CHECK(!ex.result().extended_error().message().empty());
CHECK(ex.result().extended_error().details().size() > 0);
}
}
TEST_FIXTURE(page_blob_test_base, page_clear)
{
std::vector<uint8_t> buffer;
buffer.resize(16 * 1024);
azure::storage::blob_request_options options;
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
m_blob.upload_from_stream(stream, 0, azure::storage::access_condition(), options, m_context);
std::vector<azure::storage::page_range> pages;
pages.push_back(azure::storage::page_range(0, 512 - 1));
pages.push_back(azure::storage::page_range(512 * 2, 10 * 1024 - 1));
pages.push_back(azure::storage::page_range(13 * 1024, buffer.size() - 1));
for (size_t i = 1; i < pages.size(); i++)
{
int64_t start_offset = pages[i - 1].end_offset() + 1;
int64_t length = pages[i].start_offset() - start_offset;
m_blob.clear_pages(start_offset, length, azure::storage::access_condition(), options, m_context);
}
check_page_ranges_equal(pages);
}
TEST_FIXTURE(block_blob_test_base, blob_read_stream_seek)
{
azure::storage::blob_request_options options;
options.set_stream_read_size_in_bytes(2 * 1024 * 1024);
std::vector<uint8_t> buffer;
buffer.resize(3 * 1024 * 1024);
fill_buffer_and_get_md5(buffer);
m_blob.upload_from_stream(concurrency::streams::bytestream::open_istream(buffer), azure::storage::access_condition(), options, m_context);
auto stream = m_blob.open_read(azure::storage::access_condition(), options, m_context);
CHECK(stream.can_seek());
// Because container create, blob upload, and HEAD are also in the request results,
// number of requests should start with 3.
size_t attempts = 3;
size_t position = 0;
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 1024);
attempts++;
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 512, 512);
CHECK_EQUAL(position, stream.tell());
position = buffer.size() - 128;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 128);
attempts++;
CHECK_EQUAL(position, stream.tell());
position = 4096;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 1024);
attempts++;
CHECK_EQUAL(position, stream.tell());
position += 4096;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 1024);
CHECK_EQUAL(position, stream.tell());
position -= 4096;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 128, 128);
CHECK_EQUAL(position, stream.tell());
position = 2 * 1024 * 1024 + 4096 - 512;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 512);
attempts++;
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 1024);
CHECK_EQUAL(position, stream.tell());
position -= 1024;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 2048, 2048);
CHECK_EQUAL(position, stream.tell());
position = buffer.size() - 128;
stream.seek(position);
CHECK_EQUAL(position, stream.tell());
position += seek_read_and_compare(stream, buffer, position, 1024, 128);
CHECK_EQUAL(position, stream.tell());
CHECK_EQUAL(attempts, m_context.request_results().size());
}
TEST_FIXTURE(append_blob_test_base, append_block)
{
const size_t buffer_size = 16 * 1024;
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
azure::storage::blob_request_options options;
utility::string_t md5_header;
m_context.set_sending_request([&md5_header](web::http::http_request& request, azure::storage::operation_context)
{
if (!request.headers().match(web::http::header_names::content_md5, md5_header))
{
md5_header.clear();
}
});
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
options.set_use_transactional_md5(false);
for (uint16_t i = 0; i < 3; ++i)
{
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(utility::string_t(), md5_header);
CHECK_EQUAL(i * buffer_size, offset);
CHECK_EQUAL(i + 1, m_blob.properties().append_blob_committed_block_count());
}
options.set_use_transactional_md5(false);
for (uint16_t i = 3; i < 6; ++i)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, md5, azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(md5, md5_header);
CHECK_EQUAL(i * buffer_size, offset);
CHECK_EQUAL(i + 1, m_blob.properties().append_blob_committed_block_count());
}
options.set_use_transactional_md5(true);
for (uint16_t i = 6; i < 9; ++i)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(md5, md5_header);
CHECK_EQUAL(i * buffer_size, offset);
CHECK_EQUAL(i + 1, m_blob.properties().append_blob_committed_block_count());
}
// block stream with length = 0
options.set_use_transactional_md5(true);
fill_buffer_and_get_md5(buffer);
auto stream1 = concurrency::streams::bytestream::open_istream(buffer);
stream1.seek(buffer.size());
CHECK_THROW(m_blob.append_block(stream1, utility::string_t(), azure::storage::access_condition(), options, m_context), azure::storage::storage_exception);
options.set_use_transactional_md5(true);
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.append_block(stream, dummy_md5, azure::storage::access_condition(), options, m_context), azure::storage::storage_exception);
CHECK_UTF8_EQUAL(dummy_md5, md5_header);
m_context.set_sending_request(std::function<void(web::http::http_request &, azure::storage::operation_context)>());
}
TEST_FIXTURE(append_blob_test_base, append_blob_append)
{
const size_t file_buffer_size = 24 * 1024 * 1024 + 6;
std::vector<uint8_t> file_buffer;
file_buffer.resize(file_buffer_size);
azure::storage::blob_request_options options;
options.set_use_transactional_md5(false);
utility::string_t md5_header;
m_context.set_sending_request([&md5_header](web::http::http_request& request, azure::storage::operation_context)
{
if (!request.headers().match(web::http::header_names::content_md5, md5_header))
{
md5_header.clear();
}
});
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
int block_count = 0;
// append stream (4M, 4M)
const size_t buffer_offsets1[2] = { 0, 4 * 1024 * 1024};
for (uint16_t i = 0; i < 2; ++i)
{
std::vector<uint8_t> buffer;
buffer.resize(4 * 1024 * 1024);
fill_buffer_and_get_md5(buffer);
std::copy(buffer.begin(), buffer.end(), file_buffer.begin() + buffer_offsets1[i]);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
azure::storage::access_condition condition = azure::storage::access_condition::generate_if_append_position_equal_condition(buffer_offsets1[i]);
m_blob.append_from_stream(stream, condition, options, m_context);
CHECK_UTF8_EQUAL(utility::string_t(), md5_header);
block_count++;
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
}
// append stream with length (2M, 2M, 2M)
const size_t buffer_offsets2[3] = { 8 * 1024 * 1024, 10 * 1024 * 1024, 12 * 1024 * 1024 };
for (uint16_t i = 0; i < 3; ++i)
{
std::vector<uint8_t> buffer;
buffer.resize(4 * 1024 * 1024);
fill_buffer_and_get_md5(buffer);
std::copy(buffer.begin(), buffer.begin() + 2 * 1024 * 1024, file_buffer.begin() + buffer_offsets2[i]);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
m_blob.append_from_stream(stream, 2 * 1024 * 1024, azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(utility::string_t(), md5_header);
block_count++;
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
}
// append file (5M, 5M)
const size_t buffer_offsets3[2] = { 14 * 1024 * 1024, 19 * 1024 * 1024 };
for (uint16_t i = 0; i < 2; ++i)
{
std::vector<uint8_t> buffer;
buffer.resize(5 * 1024 * 1024);
fill_buffer_and_get_md5(buffer);
std::copy(buffer.begin(), buffer.end(), file_buffer.begin() + buffer_offsets3[i]);
// create a temporary test file
utility::string_t tmp_file_path = get_random_container_name(8);
auto stream = concurrency::streams::file_stream<uint8_t>::open_ostream(tmp_file_path).get();
stream.streambuf().putn_nocopy(buffer.data(), buffer.size()).wait();
stream.close().wait();
// append from file
m_blob.append_from_file(tmp_file_path, azure::storage::access_condition(), options, m_context);
// remote the temporary test file
std::remove(utility::conversions::to_utf8string(tmp_file_path).c_str());
block_count += 2;
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
}
// append text (1, 5)
const size_t buffer_offsets4[2] = { 24 * 1024 * 1024, 24 * 1024 * 1024 + 1};
{
utility::string_t text1 = U("1");
std::string text1_copy = utility::conversions::to_utf8string(text1);
std::copy(text1_copy.begin(), text1_copy.end(), file_buffer.begin() + buffer_offsets4[0]);
m_blob.append_text(text1);
block_count++;
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
utility::string_t text2 = U("test2");
std::string text2_copy = utility::conversions::to_utf8string(text2);
std::copy(text2_copy.begin(), text2_copy.end(), file_buffer.begin() + buffer_offsets4[1]);
m_blob.append_text(text2);
block_count++;
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
}
// download the blob
concurrency::streams::container_buffer<std::vector<uint8_t>> downloaded_blob;
m_blob.download_to_stream(downloaded_blob.create_ostream(), azure::storage::access_condition(), options, m_context);
CHECK_ARRAY_EQUAL(file_buffer, downloaded_blob.collection(), (int)file_buffer.size());
m_blob.delete_blob();
m_context.set_sending_request(std::function<void(web::http::http_request &, azure::storage::operation_context)>());
}
TEST_FIXTURE(append_blob_test_base, append_block_max_size_condition)
{
const size_t buffer_size = 64 * 1024;
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
fill_buffer_and_get_md5(buffer);
azure::storage::blob_request_options options;
options.set_use_transactional_md5(false);
int64_t max_sizes1[] = {1, 1024, buffer_size - 1};
for (int64_t max_size : max_sizes1)
{
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition::generate_if_max_size_less_than_or_equal_condition(max_size), options, m_context), azure::storage::storage_exception);
m_blob.delete_blob();
}
int64_t max_sizes2[] = { buffer_size, 2 * buffer_size, std::numeric_limits<int64_t>::max() };
for (int64_t max_size : max_sizes2)
{
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition::generate_if_max_size_less_than_or_equal_condition(max_size), options, m_context);
CHECK_EQUAL(0, offset);
m_blob.delete_blob();
}
int64_t blob_size = 0;
int block_count = 0;
m_blob.create_or_replace(azure::storage::access_condition(), options, m_context);
for (uint16_t i = 0; i < 3; ++i)
{
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition(), options, m_context);
block_count++;
CHECK_EQUAL(blob_size, offset);
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
blob_size += buffer.size();
}
int64_t max_sizes3[] = { 3 * buffer_size - 1, 3 * buffer_size, 3 * buffer_size + 1, 4 * buffer_size - 1 };
for (int64_t max_size : max_sizes3)
{
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition::generate_if_max_size_less_than_or_equal_condition(max_size), options, m_context), azure::storage::storage_exception);
}
int64_t max_sizes4[] = { 4 * buffer_size, std::numeric_limits<int64_t>::max() };
for (int64_t max_size : max_sizes4)
{
auto stream = concurrency::streams::bytestream::open_istream(buffer);
int64_t offset = m_blob.append_block(stream, utility::string_t(), azure::storage::access_condition::generate_if_max_size_less_than_or_equal_condition(max_size), options, m_context);
block_count++;
CHECK_EQUAL(blob_size, offset);
CHECK_EQUAL(block_count, m_blob.properties().append_blob_committed_block_count());
blob_size += buffer.size();
}
m_blob.delete_blob();
}
TEST_FIXTURE(container_test_base, container_list_blobs)
{
m_container.create(azure::storage::blob_container_public_access_type::off, azure::storage::blob_request_options(), m_context);
check_container_no_stale_property(m_container);
std::map<utility::string_t, azure::storage::cloud_blob> blobs;
for (int i = 0; i < 4; i++)
{
auto index = utility::conversions::print_string(i);
auto blob = m_container.get_block_blob_reference(_XPLATSTR("blockblob") + index);
blob.metadata()[_XPLATSTR("index")] = index;
std::vector<uint8_t> buffer;
buffer.resize(i * 16 * 1024);
auto stream = concurrency::streams::container_stream<std::vector<uint8_t>>::open_istream(buffer);
blob.upload_from_stream(stream, azure::storage::access_condition(), azure::storage::blob_request_options(), m_context);
blobs[blob.name()] = blob;
}
for (int i = 0; i < 3; i++)
{
auto index = utility::conversions::print_string(i);
auto blob = m_container.get_page_blob_reference(_XPLATSTR("pageblob") + index);
blob.metadata()[_XPLATSTR("index")] = index;
blob.create(i * 512, 0, azure::storage::access_condition(), azure::storage::blob_request_options(), m_context);
check_container_no_stale_property(m_container);
blobs[blob.name()] = blob;
}
for (int i = 0; i < 3; i++)
{
auto index = utility::conversions::print_string(i);
auto blob = m_container.get_append_blob_reference(_XPLATSTR("appendblob") + index);
blob.metadata()[_XPLATSTR("index")] = index;
blob.create_or_replace(azure::storage::access_condition(), azure::storage::blob_request_options(), m_context);
check_container_no_stale_property(m_container);
std::vector<uint8_t> buffer;
buffer.resize((i + 1) * 8 * 1024);
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::container_stream<std::vector<uint8_t>>::open_istream(buffer);
blob.append_block(stream, utility::string_t(), azure::storage::access_condition(), azure::storage::blob_request_options(), m_context);
blobs[blob.name()] = blob;
}
auto listing1 = list_all_blobs(utility::string_t(), azure::storage::blob_listing_details::all, 0, azure::storage::blob_request_options());
for (auto iter = listing1.begin(); iter != listing1.end(); ++iter)
{
auto blob = blobs.find(iter->name());
CHECK(blob != blobs.end());
CHECK_UTF8_EQUAL(blob->second.uri().primary_uri().to_string(), iter->uri().primary_uri().to_string());
CHECK_UTF8_EQUAL(blob->second.uri().secondary_uri().to_string(), iter->uri().secondary_uri().to_string());
auto index_str = blob->second.metadata().find(_XPLATSTR("index"));
CHECK(index_str != blob->second.metadata().end());
auto index = utility::conversions::scan_string<int>(index_str->second);
switch (iter->type())
{
case azure::storage::blob_type::block_blob:
CHECK_EQUAL(index * 16 * 1024, iter->properties().size());
break;
case azure::storage::blob_type::page_blob:
CHECK_EQUAL(index * 512, iter->properties().size());
break;
case azure::storage::blob_type::append_blob:
CHECK_EQUAL((index + 1) * 8 * 1024, iter->properties().size());
break;
default:
CHECK(false);
break;
}
blobs.erase(blob);
}
CHECK_EQUAL(0U, blobs.size());
auto listing2 = list_all_blobs(_XPLATSTR("block"), azure::storage::blob_listing_details::none, 10, azure::storage::blob_request_options());
CHECK_EQUAL(4U, listing2.size());
for (auto iter = listing2.begin(); iter != listing2.end(); ++iter)
{
CHECK(iter->metadata().empty());
}
}
utility::string_t blob_service_test_base::fill_buffer_and_get_md5(std::vector<uint8_t>& buffer)
{
return fill_buffer_and_get_md5(buffer, 0, buffer.size());
}
TEST_FIXTURE(page_blob_test_base, page_upload)
{
std::vector<uint8_t> buffer;
buffer.resize(512);
azure::storage::blob_request_options options;
std::vector<azure::storage::page_range> pages;
utility::string_t md5_header;
m_context.set_sending_request([&md5_header] (web::http::http_request& request, azure::storage::operation_context)
{
if (!request.headers().match(web::http::header_names::content_md5, md5_header))
{
md5_header.clear();
}
});
m_blob.create(12 * 1024 * 1024, 0, azure::storage::access_condition(), options, m_context);
options.set_use_transactional_md5(false);
for (int i = 0; i < 3; ++i)
{
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
azure::storage::page_range range(i * 1024, i * 1024 + buffer.size() - 1);
pages.push_back(range);
m_blob.upload_pages(stream, range.start_offset(), utility::string_t(), azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(utility::string_t(), md5_header);
}
check_page_ranges_equal(pages);
options.set_use_transactional_md5(false);
for (int i = 3; i < 6; ++i)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
azure::storage::page_range range(i * 1536, i * 1536 + buffer.size() - 1);
pages.push_back(range);
m_blob.upload_pages(stream, range.start_offset(), md5, azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(md5, md5_header);
}
check_page_ranges_equal(pages);
options.set_use_transactional_md5(true);
for (int i = 6; i < 9; ++i)
{
auto md5 = fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
azure::storage::page_range range(i * 2048, i * 2048 + buffer.size() - 1);
pages.push_back(range);
m_blob.upload_pages(stream, range.start_offset(), utility::string_t(), azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(md5, md5_header);
}
options.set_use_transactional_md5(false);
{
// upload a page range of max_block_size
std::vector<uint8_t> big_buffer;
big_buffer.resize(azure::storage::protocol::max_block_size);
auto md5 = fill_buffer_and_get_md5(big_buffer);
auto stream = concurrency::streams::bytestream::open_istream(big_buffer);
azure::storage::page_range range(4 * 1024 * 1024, 4 * 1024 * 1024 + azure::storage::protocol::max_block_size - 1);
pages.push_back(range);
m_blob.upload_pages(stream, range.start_offset(), md5, azure::storage::access_condition(), options, m_context);
CHECK_UTF8_EQUAL(md5, md5_header);
}
check_page_ranges_equal(pages);
options.set_use_transactional_md5(true);
fill_buffer_and_get_md5(buffer);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.upload_pages(stream, 0, dummy_md5, azure::storage::access_condition(), options, m_context), azure::storage::storage_exception);
CHECK_UTF8_EQUAL(dummy_md5, md5_header);
// trying upload page ranges bigger than max_block_size
{
buffer.resize(azure::storage::protocol::max_block_size + 1);
fill_buffer_and_get_md5(buffer);
azure::storage::page_range range(8 * 1024 * 1024, 8 * 1024 * 1024 + azure::storage::protocol::max_block_size -1);
auto stream = concurrency::streams::bytestream::open_istream(buffer);
CHECK_THROW(m_blob.upload_pages(stream, range.start_offset(), utility::string_t(), azure::storage::access_condition(), options, m_context), std::invalid_argument);
}
check_page_ranges_equal(pages);
m_context.set_sending_request(std::function<void(web::http::http_request &, azure::storage::operation_context)>());
}