static void generate_index_ir(NODE *node, IR *ir, int result) { int offset = ir_make_temp(ir); int base = ir_make_temp(ir); SYMBOL *sym = ast_get_symbol(S(node).identifier); size_t size = sym ? symbol_size(sym) : INTEGER_SIZE; ast_generate_ir(S(node).identifier, ir); ast_generate_ir(S(node).count, ir); ir_add(ir, IR_REF, ast_ir_type(S(node).identifier), ast_ir_value(S(node).identifier), IR_TEMP, &base); ir_add(ir, IR_MULTIPLY, ast_ir_type(S(node).count), ast_ir_value(S(node).count), IR_CONST, &size, IR_TEMP, &offset); ir_add(ir, IR_ADD, IR_TEMP, &base, IR_TEMP, &offset, IR_TEMP, &result); }
TEST(TestTraceLinearBlockDecoder, print) { typedef fifi::binary field_type; typedef kodo::full_rlnc_decoder<field_type, kodo::enable_trace> decoder_type; uint32_t symbols = 3; uint32_t symbol_size = 10; decoder_type::factory decoder_factory(symbols, symbol_size); auto decoder = decoder_factory.build(); // Check that this code compiles { std::stringstream out; decoder->print_decoder_state(out); std::string result = "000 ?: 0 0 0 \n" "001 ?: 0 0 0 \n" "002 ?: 0 0 0 \n"; EXPECT_EQ(result, out.str()); } // Create a empty symbol and put it in std::vector<uint8_t> coefficients( decoder->coefficient_vector_size(), '\0'); std::vector<uint8_t> symbol( decoder->symbol_size(), '\0'); decoder->decode_symbol(&symbol[0], &coefficients[0]); { std::stringstream out; decoder->print_decoder_state(out); std::string result = "000 ?: 0 0 0 \n" "001 ?: 0 0 0 \n" "002 ?: 0 0 0 \n"; EXPECT_EQ(result, out.str()); } // Create a dummy symbol and put it in std::fill(coefficients.begin(), coefficients.end(), '\0'); fifi::set_value<field_type>(&coefficients[0], 1, 1U); fifi::set_value<field_type>(&coefficients[0], 2, 1U); std::fill(symbol.begin(), symbol.end(), 'a'); decoder->decode_symbol(&symbol[0], &coefficients[0]); { std::stringstream out; decoder->print_decoder_state(out); std::string result = "000 ?: 0 0 0 \n" "001 C: 0 1 1 \n" "002 ?: 0 0 0 \n"; EXPECT_EQ(result, out.str()); } std::fill(symbol.begin(), symbol.end(), 'b'); decoder->decode_symbol(&symbol[0], 0U); { std::stringstream out; decoder->print_decoder_state(out); std::string result = "000 U: 1 0 0 \n" "001 C: 0 1 1 \n" "002 ?: 0 0 0 \n"; EXPECT_EQ(result, out.str()); } }
// thread function to wait for the decoders to finish decoding and // print output. Only used when decoding static void print_output (struct write_out_args args) { size_t bytes_left = args.bytes; size_t current_block = 0; uint16_t last_symbol = 0; std::unique_lock<std::mutex> lock (*args.mtx, std::defer_lock); while (*args.status == Out_Status::WORKING || *args.status == Out_Status::GRACEFUL_STOP) { lock.lock(); auto dec_it = args.decoders->find (current_block); if (dec_it == args.decoders->end()) { if (*args.status == Out_Status::GRACEFUL_STOP) { lock.unlock(); break; } args.cond->wait (lock); lock.unlock(); continue; } auto dec = dec_it->second.get(); if (!dec->can_decode()) { if (*args.status == Out_Status::GRACEFUL_STOP) { *args.status = Out_Status::ERROR; return; } args.cond->wait (lock); lock.unlock(); continue; } lock.unlock(); auto pair = dec->wait_sync(); if (pair.first == RaptorQ__v1::Error::NEED_DATA) { if (*args.status == Out_Status::GRACEFUL_STOP) { lock.lock(); if (dec->poll().first == RaptorQ__v1::Error::NONE) continue; *args.status = Out_Status::ERROR; return; } continue; } if (pair.first != RaptorQ__v1::Error::NONE) { // internal error or interrupted computation lock.lock(); *args.status = Out_Status::ERROR; return; } size_t sym_size = dec->symbol_size(); std::vector<uint8_t> buffer (sym_size);; for (; last_symbol < pair.second; ++last_symbol) { buffer.clear(); buffer.insert (buffer.begin(), sym_size, 0); auto buf_start = buffer.begin(); auto to_write = dec->decode_symbol (buf_start, buffer.end(), last_symbol); if (to_write != RaptorQ__v1::Error::NONE) { std::cerr << "ERR: partial or empty symbol from decoder\n"; lock.lock(); *args.status = Out_Status::ERROR; return; } size_t writes_left = std::min (bytes_left, static_cast<size_t> (sym_size)); #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wshorten-64-to-32" args.output->write (reinterpret_cast<char *> (buffer.data()), static_cast<int64_t> (writes_left)); #pragma clang diagnostic pop bytes_left -= writes_left; if (bytes_left == 0) { // early exit. we wrote everything. last_symbol = dec->symbols(); break; } } if (last_symbol == dec->symbols()) { lock.lock(); dec_it = args.decoders->find (current_block); dec_it->second.reset (nullptr); lock.unlock(); ++current_block; last_symbol = 0; } } lock.lock(); *args.status = Out_Status::EXITED; }
/// @copydoc block_partitioning::block_size(uint32_t) const uint32_t block_size(uint32_t block_id) const { assert(m_total_blocks > block_id); return symbols(block_id) * symbol_size(block_id); }
int main() { //! [2] // Set the number of symbols (i.e. the generation size in RLNC // terminology) and the size of a symbol in bytes uint32_t max_symbols = 40; uint32_t max_symbol_size = 64; uint32_t object_size = 6400; //! [3] //! [4] using storage_encoder = kodo::object::storage_encoder< kodo::shallow_full_rlnc_encoder<fifi::binary>, kodo::fixed_partitioning_scheme>; using storage_decoder = kodo::object::storage_decoder< kodo::shallow_full_rlnc_decoder<fifi::binary>, kodo::fixed_partitioning_scheme>; //! [5] storage_encoder::factory encoder_factory(max_symbols, max_symbol_size); storage_decoder::factory decoder_factory(max_symbols, max_symbol_size); std::vector<uint8_t> data_out(object_size, '\0'); std::vector<uint8_t> data_in(object_size, 'x'); encoder_factory.set_storage(sak::storage(data_in)); decoder_factory.set_storage(sak::storage(data_out)); auto object_encoder = encoder_factory.build(); auto object_decoder = decoder_factory.build(); std::cout << "Object size = " << object_size << std::endl; std::cout << "Encoder blocks = " << object_encoder->blocks() << std::endl; std::cout << "Decoder blocks = " << object_decoder->blocks() << std::endl; //! [6] for (uint32_t i = 0; i < object_encoder->blocks(); ++i) { std::cout << "Block = " << i << " symbols = " << object_encoder->symbols(i) << " symbol size = " << object_encoder->symbol_size(i) << std::endl; } //! [7] for (uint32_t i = 0; i < object_encoder->blocks(); ++i) { auto e = object_encoder->build(i); auto d = object_decoder->build(i); std::vector<uint8_t> payload(e->payload_size()); while (!d->is_complete()) { e->encode( payload.data() ); // Here we would send and receive the payload over a // network. Lets throw away some packet to simulate. if (rand() % 2) { continue; } d->decode( payload.data() ); } } // Check we properly decoded the data if (data_in == data_out) { std::cout << "Data decoded correctly" << std::endl; } else { std::cout << "Unexpected failure to decode " << "please file a bug report :)" << std::endl; } }
inline void test_basic_api(uint32_t symbols, uint32_t symbol_size) { // Common setting typename Encoder::factory encoder_factory(symbols, symbol_size); auto encoder = encoder_factory.build(); typename Decoder::factory decoder_factory(symbols, symbol_size); auto decoder = decoder_factory.build(); EXPECT_TRUE(symbols == encoder_factory.max_symbols()); EXPECT_TRUE(symbol_size == encoder_factory.max_symbol_size()); EXPECT_TRUE(symbols == encoder->symbols()); EXPECT_TRUE(symbol_size == encoder->symbol_size()); EXPECT_TRUE(symbols == decoder_factory.max_symbols()); EXPECT_TRUE(symbol_size == decoder_factory.max_symbol_size()); EXPECT_TRUE(symbols == decoder->symbols()); EXPECT_TRUE(symbol_size == decoder->symbol_size()); EXPECT_TRUE(encoder->symbol_length() > 0); EXPECT_TRUE(decoder->symbol_length() > 0); EXPECT_TRUE(encoder->block_size() == symbols * symbol_size); EXPECT_TRUE(decoder->block_size() == symbols * symbol_size); EXPECT_TRUE(encoder_factory.max_payload_size() >= encoder->payload_size()); EXPECT_TRUE(decoder_factory.max_payload_size() >= decoder->payload_size()); EXPECT_EQ(encoder_factory.max_payload_size(), decoder_factory.max_payload_size()); // Encode/decode operations EXPECT_EQ(encoder->payload_size(), decoder->payload_size()); std::vector<uint8_t> payload(encoder->payload_size()); std::vector<uint8_t> data_in = random_vector(encoder->block_size()); std::vector<uint8_t> data_in_copy(data_in); sak::mutable_storage storage_in = sak::storage(data_in); sak::mutable_storage storage_in_copy = sak::storage(data_in_copy); EXPECT_TRUE(sak::equal(storage_in, storage_in_copy)); // Only used for prime fields, lets reconsider how we implement // this less intrusive uint32_t prefix = 0; if(fifi::is_prime2325<typename Encoder::field_type>::value) { // This field only works for multiple of uint32_t assert((encoder->block_size() % 4) == 0); uint32_t block_length = encoder->block_size() / 4; fifi::prime2325_binary_search search(block_length); prefix = search.find_prefix(storage_in_copy); // Apply the negated prefix fifi::apply_prefix(storage_in_copy, ~prefix); } encoder->set_symbols(storage_in_copy); // Set the encoder non-systematic if(kodo::is_systematic_encoder(encoder)) kodo::set_systematic_off(encoder); while( !decoder->is_complete() ) { uint32_t payload_used = encoder->encode( &payload[0] ); EXPECT_TRUE(payload_used <= encoder->payload_size()); decoder->decode( &payload[0] ); } std::vector<uint8_t> data_out(decoder->block_size(), '\0'); decoder->copy_symbols(sak::storage(data_out)); if(fifi::is_prime2325<typename Encoder::field_type>::value) { // Now we have to apply the negated prefix to the decoded data fifi::apply_prefix(sak::storage(data_out), ~prefix); } EXPECT_TRUE(std::equal(data_out.begin(), data_out.end(), data_in.begin())); }
inline void run_test_on_the_fly_systematic_no_errors(uint32_t symbols, uint32_t symbol_size) { // Common setting typename Encoder::factory encoder_factory(symbols, symbol_size); auto encoder = encoder_factory.build(); typename Decoder::factory decoder_factory(symbols, symbol_size); auto decoder = decoder_factory.build(); // Encode/decode operations EXPECT_TRUE(encoder->payload_size() == decoder->payload_size()); std::vector<uint8_t> payload(encoder->payload_size()); std::vector<uint8_t> data_in = random_vector(encoder->block_size()); auto symbol_sequence = sak::split_storage( sak::storage(data_in), symbol_size); // Make sure the encoder is systematic if(kodo::has_systematic_encoder<Encoder>::value) kodo::set_systematic_on(encoder); while( !decoder->is_complete() ) { encoder->encode( &payload[0] ); decoder->decode( &payload[0] ); if(decoder->rank() > 0) { EXPECT_TRUE(kodo::is_partial_complete(decoder)); } if(kodo::is_partial_complete(decoder)) { // Check that we as many pivot elements as expected and that these // are decoded uint32_t symbols_uncoded = 0; for(uint32_t i = 0; i < decoder->symbols(); ++i) { if(!decoder->is_symbol_uncoded(i)) continue; ++symbols_uncoded; auto symbol_storage = sak::storage(decoder->symbol(i), decoder->symbol_size()); EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i])); } EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded()); } // set symbol 50% of the time ONLY if rank is not full if(encoder->rank() < symbols) { uint32_t i = encoder->rank(); encoder->set_symbol(i, symbol_sequence[i]); } EXPECT_TRUE(encoder->rank() >= decoder->rank()); } std::vector<uint8_t> data_out(decoder->block_size(), '\0'); decoder->copy_symbols(sak::storage(data_out)); EXPECT_TRUE(std::equal(data_out.begin(), data_out.end(), data_in.begin())); }
inline void run_test_on_the_fly(uint32_t symbols, uint32_t symbol_size) { // Common setting typename Encoder::factory encoder_factory(symbols, symbol_size); auto encoder = encoder_factory.build(); typename Decoder::factory decoder_factory(symbols, symbol_size); auto decoder = decoder_factory.build(); std::vector<uint8_t> payload(encoder->payload_size()); std::vector<uint8_t> data_in = random_vector(encoder->block_size()); auto symbol_sequence = sak::split_storage( sak::storage(data_in), symbol_size); // Set the encoder non-systematic if(kodo::has_systematic_encoder<Encoder>::value) kodo::set_systematic_off(encoder); EXPECT_EQ(encoder->rank(), 0U); EXPECT_EQ(decoder->rank(), 0U); while( !decoder->is_complete() ) { EXPECT_TRUE(encoder->rank() >= decoder->rank()); encoder->encode( &payload[0] ); // Simulate some loss if((rand() % 2) == 0) continue; decoder->decode( &payload[0] ); if(kodo::is_partial_complete(decoder)) { // Check that we as many pivot elements as expected and that these // are decoded uint32_t symbols_uncoded = 0; for(uint32_t i = 0; i < decoder->symbols(); ++i) { if(!decoder->is_symbol_uncoded(i)) continue; ++symbols_uncoded; auto symbol_storage = sak::storage(decoder->symbol(i), decoder->symbol_size()); EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i])); } EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded()); } if(((rand() % 2) == 0) && encoder->rank() < symbols) { uint32_t i = encoder->rank(); encoder->set_symbol(i, symbol_sequence[i]); } } std::vector<uint8_t> data_out(decoder->block_size(), '\0'); decoder->copy_symbols(sak::storage(data_out)); EXPECT_TRUE(std::equal(data_out.begin(), data_out.end(), data_in.begin())); }
inline void run_test_on_the_fly_systematic(uint32_t symbols, uint32_t symbol_size) { // Common setting typename Encoder::factory encoder_factory(symbols, symbol_size); auto encoder = encoder_factory.build(); typename Decoder::factory decoder_factory(symbols, symbol_size); auto decoder = decoder_factory.build(); // Encode/decode operations EXPECT_TRUE(encoder->payload_size() == decoder->payload_size()); std::vector<uint8_t> payload(encoder->payload_size()); std::vector<uint8_t> data_in = random_vector(encoder->block_size()); auto symbol_sequence = sak::split_storage( sak::storage(data_in), symbol_size); // Make sure the encoder is systematic if(kodo::has_systematic_encoder<Encoder>::value) kodo::set_systematic_on(encoder); while( !decoder->is_complete() ) { encoder->encode( &payload[0] ); // Simulate some loss if((rand() % 2) == 0) continue; uint32_t old_rank = decoder->rank(); decoder->decode( &payload[0] ); if(decoder->rank() > old_rank) { // We has a rank increase if(encoder->rank() == decoder->rank()) { // The rank of the encoder matches the decoder, if // this unit test fails you most likely do not have a // layer which updates the decoding symbol status when // rank matches. E.g. look at: // rank_symbol_decoding_status_updater.hpp EXPECT_TRUE(kodo::is_partial_complete(decoder)); } } if(kodo::is_partial_complete(decoder)) { // Check that we as many pivot elements as expected and that these // are decoded uint32_t symbols_uncoded = 0; for(uint32_t i = 0; i < decoder->symbols(); ++i) { if(!decoder->is_symbol_uncoded(i)) continue; ++symbols_uncoded; auto symbol_storage = sak::storage(decoder->symbol(i), decoder->symbol_size()); EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i])); } EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded()); } // set symbol 50% of the time ONLY if rank is not full if(((rand() % 2) == 0) && (encoder->rank() < symbols)) { uint32_t i = encoder->rank(); encoder->set_symbol(i, symbol_sequence[i]); } EXPECT_TRUE(encoder->rank() >= decoder->rank()); } std::vector<uint8_t> data_out(decoder->block_size(), '\0'); decoder->copy_symbols(sak::storage(data_out)); EXPECT_TRUE(std::equal(data_out.begin(), data_out.end(), data_in.begin())); }
// Tests that encoding and decoding a file withe the file encoder // works. TEST(TestFileEncoder, test_file_encoder) { { std::string encode_filename = "encode-file"; std::string decode_filename = "decode-file"; // Write a test file std::ofstream encode_file; encode_file.open (encode_filename, std::ios::binary); uint32_t size = 500; for(uint32_t i = 0; i < size; ++i) { char c = rand() % 255; encode_file.write(&c, 1); } encode_file.close(); typedef kodo::full_rlnc_encoder<fifi::binary> encoder_t; typedef kodo::full_rlnc_decoder<fifi::binary> decoder_t; typedef kodo::file_encoder<encoder_t> file_encoder_t; typedef kodo::object_decoder<decoder_t> object_decoder_t; uint32_t max_symbols = 10; uint32_t max_symbol_size = 10; file_encoder_t::factory encoder_factory( max_symbols, max_symbol_size); file_encoder_t file_encoder(encoder_factory, encode_filename); object_decoder_t::factory decoder_factory( max_symbols, max_symbol_size); object_decoder_t object_decoder(decoder_factory, size); EXPECT_EQ(object_decoder.decoders(), file_encoder.encoders()); // Open the decode file std::ofstream decode_file; decode_file.open (decode_filename, std::ios::binary); for(uint32_t i = 0; i < file_encoder.encoders(); ++i) { auto encoder = file_encoder.build(i); auto decoder = object_decoder.build(i); EXPECT_EQ(encoder->symbols(), decoder->symbols()); EXPECT_EQ(encoder->symbol_size(), decoder->symbol_size()); EXPECT_EQ(encoder->bytes_used(), decoder->bytes_used()); // Set the encoder non-systematic if(kodo::is_systematic_encoder(encoder)) kodo::set_systematic_off(encoder); std::vector<uint8_t> payload(encoder->payload_size()); while( !decoder->is_complete() ) { // Encode a packet into the payload buffer encoder->encode( &payload[0] ); // Pass that packet to the decoder decoder->decode( &payload[0] ); } std::vector<uint8_t> data_out(decoder->block_size()); decoder->copy_symbols(sak::storage(data_out)); decode_file.write( reinterpret_cast<char*>(&data_out[0]), decoder->bytes_used()); } decode_file.close(); } }