static void generate_index_ir(NODE *node, IR *ir, int result)
{
int offset = ir_make_temp(ir);
int base = ir_make_temp(ir);
SYMBOL *sym = ast_get_symbol(S(node).identifier);
size_t size = sym ? symbol_size(sym) : INTEGER_SIZE;
ast_generate_ir(S(node).identifier, ir);
ast_generate_ir(S(node).count, ir);
ir_add(ir, IR_REF,
ast_ir_type(S(node).identifier), ast_ir_value(S(node).identifier),
IR_TEMP, &base);
ir_add(ir, IR_MULTIPLY,
ast_ir_type(S(node).count), ast_ir_value(S(node).count),
IR_CONST, &size,
IR_TEMP, &offset);
ir_add(ir, IR_ADD, IR_TEMP, &base, IR_TEMP, &offset, IR_TEMP, &result);
}
TEST(TestTraceLinearBlockDecoder, print)
{
typedef fifi::binary field_type;
typedef kodo::full_rlnc_decoder<field_type, kodo::enable_trace>
decoder_type;
uint32_t symbols = 3;
uint32_t symbol_size = 10;
decoder_type::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Check that this code compiles
{
std::stringstream out;
decoder->print_decoder_state(out);
std::string result = "000 ?: 0 0 0 \n"
"001 ?: 0 0 0 \n"
"002 ?: 0 0 0 \n";
EXPECT_EQ(result, out.str());
}
// Create a empty symbol and put it in
std::vector<uint8_t> coefficients(
decoder->coefficient_vector_size(), '\0');
std::vector<uint8_t> symbol(
decoder->symbol_size(), '\0');
decoder->decode_symbol(&symbol[0], &coefficients[0]);
{
std::stringstream out;
decoder->print_decoder_state(out);
std::string result = "000 ?: 0 0 0 \n"
"001 ?: 0 0 0 \n"
"002 ?: 0 0 0 \n";
EXPECT_EQ(result, out.str());
}
// Create a dummy symbol and put it in
std::fill(coefficients.begin(), coefficients.end(), '\0');
fifi::set_value<field_type>(&coefficients[0], 1, 1U);
fifi::set_value<field_type>(&coefficients[0], 2, 1U);
std::fill(symbol.begin(), symbol.end(), 'a');
decoder->decode_symbol(&symbol[0], &coefficients[0]);
{
std::stringstream out;
decoder->print_decoder_state(out);
std::string result = "000 ?: 0 0 0 \n"
"001 C: 0 1 1 \n"
"002 ?: 0 0 0 \n";
EXPECT_EQ(result, out.str());
}
std::fill(symbol.begin(), symbol.end(), 'b');
decoder->decode_symbol(&symbol[0], 0U);
{
std::stringstream out;
decoder->print_decoder_state(out);
std::string result = "000 U: 1 0 0 \n"
"001 C: 0 1 1 \n"
"002 ?: 0 0 0 \n";
EXPECT_EQ(result, out.str());
}
}
// thread function to wait for the decoders to finish decoding and
// print output. Only used when decoding
static void print_output (struct write_out_args args)
{
size_t bytes_left = args.bytes;
size_t current_block = 0;
uint16_t last_symbol = 0;
std::unique_lock<std::mutex> lock (*args.mtx, std::defer_lock);
while (*args.status == Out_Status::WORKING ||
*args.status == Out_Status::GRACEFUL_STOP) {
lock.lock();
auto dec_it = args.decoders->find (current_block);
if (dec_it == args.decoders->end()) {
if (*args.status == Out_Status::GRACEFUL_STOP) {
lock.unlock();
break;
}
args.cond->wait (lock);
lock.unlock();
continue;
}
auto dec = dec_it->second.get();
if (!dec->can_decode()) {
if (*args.status == Out_Status::GRACEFUL_STOP) {
*args.status = Out_Status::ERROR;
return;
}
args.cond->wait (lock);
lock.unlock();
continue;
}
lock.unlock();
auto pair = dec->wait_sync();
if (pair.first == RaptorQ__v1::Error::NEED_DATA) {
if (*args.status == Out_Status::GRACEFUL_STOP) {
lock.lock();
if (dec->poll().first == RaptorQ__v1::Error::NONE)
continue;
*args.status = Out_Status::ERROR;
return;
}
continue;
}
if (pair.first != RaptorQ__v1::Error::NONE) {
// internal error or interrupted computation
lock.lock();
*args.status = Out_Status::ERROR;
return;
}
size_t sym_size = dec->symbol_size();
std::vector<uint8_t> buffer (sym_size);;
for (; last_symbol < pair.second; ++last_symbol) {
buffer.clear();
buffer.insert (buffer.begin(), sym_size, 0);
auto buf_start = buffer.begin();
auto to_write = dec->decode_symbol (buf_start, buffer.end(),
last_symbol);
if (to_write != RaptorQ__v1::Error::NONE) {
std::cerr << "ERR: partial or empty symbol from decoder\n";
lock.lock();
*args.status = Out_Status::ERROR;
return;
}
size_t writes_left = std::min (bytes_left,
static_cast<size_t> (sym_size));
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
args.output->write (reinterpret_cast<char *> (buffer.data()),
static_cast<int64_t> (writes_left));
#pragma clang diagnostic pop
bytes_left -= writes_left;
if (bytes_left == 0) {
// early exit. we wrote everything.
last_symbol = dec->symbols();
break;
}
}
if (last_symbol == dec->symbols()) {
lock.lock();
dec_it = args.decoders->find (current_block);
dec_it->second.reset (nullptr);
lock.unlock();
++current_block;
last_symbol = 0;
}
}
lock.lock();
*args.status = Out_Status::EXITED;
}
/// @copydoc block_partitioning::block_size(uint32_t) const
uint32_t block_size(uint32_t block_id) const
{
assert(m_total_blocks > block_id);
return symbols(block_id) * symbol_size(block_id);
}
int main()
{
//! [2]
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t max_symbols = 40;
uint32_t max_symbol_size = 64;
uint32_t object_size = 6400;
//! [3]
//! [4]
using storage_encoder = kodo::object::storage_encoder<
kodo::shallow_full_rlnc_encoder<fifi::binary>,
kodo::fixed_partitioning_scheme>;
using storage_decoder = kodo::object::storage_decoder<
kodo::shallow_full_rlnc_decoder<fifi::binary>,
kodo::fixed_partitioning_scheme>;
//! [5]
storage_encoder::factory encoder_factory(max_symbols, max_symbol_size);
storage_decoder::factory decoder_factory(max_symbols, max_symbol_size);
std::vector<uint8_t> data_out(object_size, '\0');
std::vector<uint8_t> data_in(object_size, 'x');
encoder_factory.set_storage(sak::storage(data_in));
decoder_factory.set_storage(sak::storage(data_out));
auto object_encoder = encoder_factory.build();
auto object_decoder = decoder_factory.build();
std::cout << "Object size = " << object_size << std::endl;
std::cout << "Encoder blocks = " << object_encoder->blocks() << std::endl;
std::cout << "Decoder blocks = " << object_decoder->blocks() << std::endl;
//! [6]
for (uint32_t i = 0; i < object_encoder->blocks(); ++i)
{
std::cout << "Block = " << i << " symbols = "
<< object_encoder->symbols(i) << " symbol size = "
<< object_encoder->symbol_size(i) << std::endl;
}
//! [7]
for (uint32_t i = 0; i < object_encoder->blocks(); ++i)
{
auto e = object_encoder->build(i);
auto d = object_decoder->build(i);
std::vector<uint8_t> payload(e->payload_size());
while (!d->is_complete())
{
e->encode( payload.data() );
// Here we would send and receive the payload over a
// network. Lets throw away some packet to simulate.
if (rand() % 2)
{
continue;
}
d->decode( payload.data() );
}
}
// Check we properly decoded the data
if (data_in == data_out)
{
std::cout << "Data decoded correctly" << std::endl;
}
else
{
std::cout << "Unexpected failure to decode "
<< "please file a bug report :)" << std::endl;
}
}
inline void test_basic_api(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
EXPECT_TRUE(symbols == encoder_factory.max_symbols());
EXPECT_TRUE(symbol_size == encoder_factory.max_symbol_size());
EXPECT_TRUE(symbols == encoder->symbols());
EXPECT_TRUE(symbol_size == encoder->symbol_size());
EXPECT_TRUE(symbols == decoder_factory.max_symbols());
EXPECT_TRUE(symbol_size == decoder_factory.max_symbol_size());
EXPECT_TRUE(symbols == decoder->symbols());
EXPECT_TRUE(symbol_size == decoder->symbol_size());
EXPECT_TRUE(encoder->symbol_length() > 0);
EXPECT_TRUE(decoder->symbol_length() > 0);
EXPECT_TRUE(encoder->block_size() == symbols * symbol_size);
EXPECT_TRUE(decoder->block_size() == symbols * symbol_size);
EXPECT_TRUE(encoder_factory.max_payload_size() >=
encoder->payload_size());
EXPECT_TRUE(decoder_factory.max_payload_size() >=
decoder->payload_size());
EXPECT_EQ(encoder_factory.max_payload_size(),
decoder_factory.max_payload_size());
// Encode/decode operations
EXPECT_EQ(encoder->payload_size(), decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
std::vector<uint8_t> data_in_copy(data_in);
sak::mutable_storage storage_in = sak::storage(data_in);
sak::mutable_storage storage_in_copy = sak::storage(data_in_copy);
EXPECT_TRUE(sak::equal(storage_in, storage_in_copy));
// Only used for prime fields, lets reconsider how we implement
// this less intrusive
uint32_t prefix = 0;
if(fifi::is_prime2325<typename Encoder::field_type>::value)
{
// This field only works for multiple of uint32_t
assert((encoder->block_size() % 4) == 0);
uint32_t block_length = encoder->block_size() / 4;
fifi::prime2325_binary_search search(block_length);
prefix = search.find_prefix(storage_in_copy);
// Apply the negated prefix
fifi::apply_prefix(storage_in_copy, ~prefix);
}
encoder->set_symbols(storage_in_copy);
// Set the encoder non-systematic
if(kodo::is_systematic_encoder(encoder))
kodo::set_systematic_off(encoder);
while( !decoder->is_complete() )
{
uint32_t payload_used = encoder->encode( &payload[0] );
EXPECT_TRUE(payload_used <= encoder->payload_size());
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
if(fifi::is_prime2325<typename Encoder::field_type>::value)
{
// Now we have to apply the negated prefix to the decoded data
fifi::apply_prefix(sak::storage(data_out), ~prefix);
}
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_on_the_fly_systematic_no_errors(uint32_t symbols,
uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
if(decoder->rank() > 0)
{
EXPECT_TRUE(kodo::is_partial_complete(decoder));
}
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
// set symbol 50% of the time ONLY if rank is not full
if(encoder->rank() < symbols)
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
EXPECT_TRUE(encoder->rank() >= decoder->rank());
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_on_the_fly(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_off(encoder);
EXPECT_EQ(encoder->rank(), 0U);
EXPECT_EQ(decoder->rank(), 0U);
while( !decoder->is_complete() )
{
EXPECT_TRUE(encoder->rank() >= decoder->rank());
encoder->encode( &payload[0] );
// Simulate some loss
if((rand() % 2) == 0)
continue;
decoder->decode( &payload[0] );
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
if(((rand() % 2) == 0) && encoder->rank() < symbols)
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_on_the_fly_systematic(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
// Simulate some loss
if((rand() % 2) == 0)
continue;
uint32_t old_rank = decoder->rank();
decoder->decode( &payload[0] );
if(decoder->rank() > old_rank)
{
// We has a rank increase
if(encoder->rank() == decoder->rank())
{
// The rank of the encoder matches the decoder, if
// this unit test fails you most likely do not have a
// layer which updates the decoding symbol status when
// rank matches. E.g. look at:
// rank_symbol_decoding_status_updater.hpp
EXPECT_TRUE(kodo::is_partial_complete(decoder));
}
}
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
// set symbol 50% of the time ONLY if rank is not full
if(((rand() % 2) == 0) && (encoder->rank() < symbols))
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
EXPECT_TRUE(encoder->rank() >= decoder->rank());
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
// Tests that encoding and decoding a file withe the file encoder
// works.
TEST(TestFileEncoder, test_file_encoder)
{
{
std::string encode_filename = "encode-file";
std::string decode_filename = "decode-file";
// Write a test file
std::ofstream encode_file;
encode_file.open (encode_filename, std::ios::binary);
uint32_t size = 500;
for(uint32_t i = 0; i < size; ++i)
{
char c = rand() % 255;
encode_file.write(&c, 1);
}
encode_file.close();
typedef kodo::full_rlnc_encoder<fifi::binary>
encoder_t;
typedef kodo::full_rlnc_decoder<fifi::binary>
decoder_t;
typedef kodo::file_encoder<encoder_t>
file_encoder_t;
typedef kodo::object_decoder<decoder_t>
object_decoder_t;
uint32_t max_symbols = 10;
uint32_t max_symbol_size = 10;
file_encoder_t::factory encoder_factory(
max_symbols, max_symbol_size);
file_encoder_t file_encoder(encoder_factory, encode_filename);
object_decoder_t::factory decoder_factory(
max_symbols, max_symbol_size);
object_decoder_t object_decoder(decoder_factory, size);
EXPECT_EQ(object_decoder.decoders(), file_encoder.encoders());
// Open the decode file
std::ofstream decode_file;
decode_file.open (decode_filename, std::ios::binary);
for(uint32_t i = 0; i < file_encoder.encoders(); ++i)
{
auto encoder = file_encoder.build(i);
auto decoder = object_decoder.build(i);
EXPECT_EQ(encoder->symbols(), decoder->symbols());
EXPECT_EQ(encoder->symbol_size(), decoder->symbol_size());
EXPECT_EQ(encoder->bytes_used(), decoder->bytes_used());
// Set the encoder non-systematic
if(kodo::is_systematic_encoder(encoder))
kodo::set_systematic_off(encoder);
std::vector<uint8_t> payload(encoder->payload_size());
while( !decoder->is_complete() )
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// Pass that packet to the decoder
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
decode_file.write(
reinterpret_cast<char*>(&data_out[0]), decoder->bytes_used());
}
decode_file.close();
}
}
