TEST(TestReedSolomonCodes, test_encode_decode)
{
{
kodo::rs_encoder<fifi::binary8>::factory encoder_factory(255, 1600);
kodo::rs_decoder<fifi::binary8>::factory decoder_factory(255, 1600);
uint32_t symbols = rand_symbols(255);
uint32_t symbol_size = rand_symbol_size();
encoder_factory.set_symbols(symbols);
encoder_factory.set_symbol_size(symbol_size);
decoder_factory.set_symbols(symbols);
decoder_factory.set_symbol_size(symbol_size);
auto encoder = encoder_factory.build();
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
encoder->set_symbols(sak::storage(data_in));
uint32_t symbol_count = 0;
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
++symbol_count;
}
// A reed solomon code should be able to decode
// with exactly k symbols
EXPECT_EQ(symbol_count, symbols);
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
}
int main()
{
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t max_symbols = 42;
uint32_t max_symbol_size = 64;
std::string encode_filename = "encode-file.bin";
// Create a test file for encoding.
std::ofstream encode_file;
encode_file.open (encode_filename, std::ios::binary);
uint32_t file_size = 50000;
std::vector<char> encode_data(file_size);
std::vector<char> decode_data;
// Just write some bytes to the file
for(uint32_t i = 0; i < file_size; ++i)
{
encode_data[i] = rand() % 255;
}
encode_file.write(&encode_data[0], file_size);
encode_file.close();
// Select the encoding and decoding algorithms
typedef kodo::full_rlnc_encoder<fifi::binary>
encoder_t;
typedef kodo::full_rlnc_decoder<fifi::binary>
decoder_t;
// Now for the encoder we use a file_encoder with the chosen
// encoding algorithm
typedef kodo::file_encoder<encoder_t>
file_encoder_t;
// For decoding we use an object_decoder with the chosen
// decoding algorithm
typedef kodo::object_decoder<decoder_t>
object_decoder_t;
// Create the encoder factory - builds the individual encoders used
file_encoder_t::factory encoder_factory(max_symbols, max_symbol_size);
// Create the actual file encoder using the encoder factory and
// the filename of the file to be encoded
file_encoder_t file_encoder(encoder_factory, encode_filename);
// Create the decoder factory - build the individual decoders used
object_decoder_t::factory decoder_factory(max_symbols, max_symbol_size);
// Create the object decoder using the decoder factory and the
// size of the file to be decoded
object_decoder_t object_decoder(decoder_factory, file_size);
// Now in the following loop we go through all the encoders
// needed to encode the entire file. We the build the corresponding
// decoder and decode the chunk immediately. In practice where
// encoders and decoders are on different devices e.g. connected
// over a network, we would have to pass also the encoder and decoder
// index between the source and sink to allow the correct data would
// passed from encoder to corresponding decoder.
for(uint32_t i = 0; i < file_encoder.encoders(); ++i)
{
auto encoder = file_encoder.build(i);
auto decoder = object_decoder.build(i);
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<encoder_t>::value)
kodo::set_systematic_off(encoder);
std::vector<uint8_t> payload(encoder->payload_size());
while( !decoder->is_complete() )
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// In practice send the payload over a network, save it to
// a file etc. Then when needed build and pass it to the decoder
// Pass that packet to the decoder
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
data_out.resize(decoder->bytes_used());
decode_data.insert(decode_data.end(),
data_out.begin(),
data_out.end());
}
// Check we properly decoded the data
if (std::equal(decode_data.begin(),
decode_data.end(), encode_data.begin()))
{
std::cout << "Data decoded correctly" << std::endl;
}
else
{
std::cout << "Unexpected failure to decode "
<< "please file a bug report :)" << std::endl;
}
}
inline void test_basic_api(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
EXPECT_TRUE(symbols == encoder_factory.max_symbols());
EXPECT_TRUE(symbol_size == encoder_factory.max_symbol_size());
EXPECT_TRUE(symbols == encoder->symbols());
EXPECT_TRUE(symbol_size == encoder->symbol_size());
EXPECT_TRUE(symbols == decoder_factory.max_symbols());
EXPECT_TRUE(symbol_size == decoder_factory.max_symbol_size());
EXPECT_TRUE(symbols == decoder->symbols());
EXPECT_TRUE(symbol_size == decoder->symbol_size());
EXPECT_TRUE(encoder->symbol_length() > 0);
EXPECT_TRUE(decoder->symbol_length() > 0);
EXPECT_TRUE(encoder->block_size() == symbols * symbol_size);
EXPECT_TRUE(decoder->block_size() == symbols * symbol_size);
EXPECT_TRUE(encoder_factory.max_payload_size() >=
encoder->payload_size());
EXPECT_TRUE(decoder_factory.max_payload_size() >=
decoder->payload_size());
EXPECT_EQ(encoder_factory.max_payload_size(),
decoder_factory.max_payload_size());
// Encode/decode operations
EXPECT_EQ(encoder->payload_size(), decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
std::vector<uint8_t> data_in_copy(data_in);
sak::mutable_storage storage_in = sak::storage(data_in);
sak::mutable_storage storage_in_copy = sak::storage(data_in_copy);
EXPECT_TRUE(sak::equal(storage_in, storage_in_copy));
// Only used for prime fields, lets reconsider how we implement
// this less intrusive
uint32_t prefix = 0;
if(fifi::is_prime2325<typename Encoder::field_type>::value)
{
// This field only works for multiple of uint32_t
assert((encoder->block_size() % 4) == 0);
uint32_t block_length = encoder->block_size() / 4;
fifi::prime2325_binary_search search(block_length);
prefix = search.find_prefix(storage_in_copy);
// Apply the negated prefix
fifi::apply_prefix(storage_in_copy, ~prefix);
}
encoder->set_symbols(storage_in_copy);
// Set the encoder non-systematic
if(kodo::is_systematic_encoder(encoder))
kodo::set_systematic_off(encoder);
while( !decoder->is_complete() )
{
uint32_t payload_used = encoder->encode( &payload[0] );
EXPECT_TRUE(payload_used <= encoder->payload_size());
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
if(fifi::is_prime2325<typename Encoder::field_type>::value)
{
// Now we have to apply the negated prefix to the decoded data
fifi::apply_prefix(sak::storage(data_out), ~prefix);
}
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void
invoke_reuse_incomplete(uint32_t symbols, uint32_t symbol_size)
{
bool do_complete;
typename Encoder::factory encoder_factory(symbols, symbol_size);
typename Decoder::factory decoder_factory(symbols, symbol_size);
// Use factory a lot of times
for (uint32_t i = 0; i < 100; ++i)
{
// Build coders
auto encoder = encoder_factory.build();
auto decoder = decoder_factory.build();
// Prepare buffers
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in(encoder->block_size());
// Fill with random data
for (auto &e: data_in)
e = rand() % 256;
// Put data in encoder
encoder->set_symbols(sak::storage(data_in));
if (rand() % 100 > 90)
{
do_complete = false;
}
else
{
do_complete = true;
}
// Start encoding/decoding
while (!decoder->is_complete())
{
encoder->encode(&payload[0]);
// Loose a packet with probability
if (rand() % 100 > 90)
continue;
decoder->decode(&payload[0]);
// Stop decoding after a while with probability
if (!do_complete && decoder->rank() == symbols - 2)
break;
}
// Check if completed decoders are correct
if (decoder->is_complete())
{
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
ASSERT_TRUE(sak::equal(sak::storage(data_out),
sak::storage(data_in)));
}
}
}
/// @example use_cached_symbol_decoder.cpp
///
/// This example shows how to use the cached symbol decoder to "extract"
/// the symbol coding coefficients and the encoded symbol data from an
/// incoming symbol.
int main()
{
// The finite field we will use in the example. You can try
// with other fields by specifying e.g. fifi::binary8 for the
// extension field 2^8
typedef fifi::binary finite_field;
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t symbols = 8;
uint32_t symbol_size = 160;
// Typdefs for the encoder/decoder type we wish to use
typedef kodo::full_rlnc_encoder<finite_field> rlnc_encoder;
typedef kodo::full_rlnc_decoder<finite_field> rlnc_decoder;
typedef kodo::symbol_info_decoder<finite_field> rlnc_info_decoder;
// In the following we will make an encoder/decoder factory.
// The factories are used to build actual encoders/decoders.
// Each stack we use have their own factories.
rlnc_encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
rlnc_decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
rlnc_info_decoder::factory info_decoder_factory(symbols, symbol_size);
auto info_decoder = info_decoder_factory.build();
// Allocate some storage for a "payload" the payload is what we would
// eventually send over a network
std::vector<uint8_t> payload(encoder->payload_size());
// Allocate some data to encode. In this case we make a buffer
// with the same size as the encoder's block size (the max.
// amount a single encoder can encode)
std::vector<uint8_t> data_in(encoder->block_size());
// Just for fun - fill the data with random data
for(auto &e: data_in)
e = rand() % 256;
// Assign the data buffer to the encoder so that we may start
// to produce encoded symbols from it
encoder->set_symbols(sak::storage(data_in));
while( !decoder->is_complete())
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// Here we "simulate" a packet loss of approximately 50%
// by dropping half of the encoded packets.
// When running this example you will notice that the initial
// symbols are received systematically (i.e. uncoded). After
// sending all symbols once uncoded, the encoder will switch
// to full coding, in which case you will see the full encoding
// vectors being sent and received.
if((rand() % 2) == 0)
continue;
// Pass the encoded packet to the info decoder. After this
// information about the coded symbol can be fetched using the
// cached_symbol_decoder API
info_decoder->decode( &payload[0] );
if(!info_decoder->cached_symbol_coded())
{
// The symbol was uncoded so we may ask the cache which of the
// original symbols we have received.
std::cout << "Symbol was uncoded, index = "
<< info_decoder->cached_symbol_index() << std::endl;
// Now we pass the data directly into our actual decoder. This is
// done using the "Codec API" directly, and not through the "Payload
// API" as we would typically do.
decoder->decode_symbol( info_decoder->cached_symbol_data(),
info_decoder->cached_symbol_index());
}
else
{
// The symbol was coded so we may ask the cache to return
// the coding coefficients used to create the encoded symbol.
std::cout << "Symbol was coded, encoding vector = ";
const uint8_t* c = info_decoder->cached_symbol_coefficients();
// We loop through the coefficient buffer and print the coefficients
for(uint32_t i = 0; i < info_decoder->symbols(); ++i)
{
std::cout << (uint32_t) fifi::get_value<finite_field>(c, i)
<< " ";
}
std::cout << std::endl;
// Pass that packet to the decoder, as with the uncoded symbols
// above we pass it directly to the "Codec API"
decoder->decode_symbol(info_decoder->cached_symbol_data(),
info_decoder->cached_symbol_coefficients());
}
}
// The decoder is complete, now copy the symbols from the decoder
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
// Check we properly decoded the data
if (std::equal(data_out.begin(), data_out.end(), data_in.begin()))
{
std::cout << "Data decoded correctly" << std::endl;
}
else
{
std::cout << "Unexpected failure to decode "
<< "please file a bug report :)" << std::endl;
}
}
inline void run_test_on_the_fly_systematic_no_errors(uint32_t symbols,
uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
if(decoder->rank() > 0)
{
EXPECT_TRUE(kodo::is_partial_complete(decoder));
}
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
// set symbol 50% of the time ONLY if rank is not full
if(encoder->rank() < symbols)
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
EXPECT_TRUE(encoder->rank() >= decoder->rank());
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_on_the_fly(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_off(encoder);
EXPECT_EQ(encoder->rank(), 0U);
EXPECT_EQ(decoder->rank(), 0U);
while( !decoder->is_complete() )
{
EXPECT_TRUE(encoder->rank() >= decoder->rank());
encoder->encode( &payload[0] );
// Simulate some loss
if((rand() % 2) == 0)
continue;
decoder->decode( &payload[0] );
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
if(((rand() % 2) == 0) && encoder->rank() < symbols)
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_on_the_fly_systematic(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
// Simulate some loss
if((rand() % 2) == 0)
continue;
uint32_t old_rank = decoder->rank();
decoder->decode( &payload[0] );
if(decoder->rank() > old_rank)
{
// We has a rank increase
if(encoder->rank() == decoder->rank())
{
// The rank of the encoder matches the decoder, if
// this unit test fails you most likely do not have a
// layer which updates the decoding symbol status when
// rank matches. E.g. look at:
// rank_symbol_decoding_status_updater.hpp
EXPECT_TRUE(kodo::is_partial_complete(decoder));
}
}
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
// set symbol 50% of the time ONLY if rank is not full
if(((rand() % 2) == 0) && (encoder->rank() < symbols))
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
EXPECT_TRUE(encoder->rank() >= decoder->rank());
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void invoke_recoding(recoding_parameters param)
{
// Common setting
typename Encoder::factory encoder_factory(
param.m_max_symbols, param.m_max_symbol_size);
encoder_factory.set_symbols(param.m_symbols);
encoder_factory.set_symbol_size(param.m_symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(
param.m_max_symbols, param.m_max_symbol_size);
decoder_factory.set_symbols(param.m_symbols);
decoder_factory.set_symbol_size(param.m_symbol_size);
auto decoder_one = decoder_factory.build();
auto decoder_two = decoder_factory.build();
// If tested with a shallow decoder we have to remember to set the
// buffers to use for the decoding
std::vector<uint8_t> buffer_decoder_one(decoder_one->block_size(), '\0');
std::vector<uint8_t> buffer_decoder_two(decoder_two->block_size(), '\0');
if(kodo::has_shallow_symbol_storage<Decoder>::value)
{
decoder_one->set_symbols(sak::storage(buffer_decoder_one));
decoder_two->set_symbols(sak::storage(buffer_decoder_two));
}
EXPECT_EQ(encoder->payload_size(), decoder_one->payload_size());
EXPECT_EQ(encoder->payload_size(), decoder_two->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
encoder->set_symbols(sak::storage(data_in));
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_off(encoder);
while( !decoder_two->is_complete() )
{
uint32_t encode_size = encoder->encode( &payload[0] );
EXPECT_TRUE(encode_size <= payload.size());
EXPECT_TRUE(encode_size > 0);
decoder_one->decode( &payload[0] );
uint32_t recode_size = decoder_one->recode( &payload[0] );
EXPECT_TRUE(recode_size <= payload.size());
EXPECT_TRUE(recode_size > 0);
decoder_two->decode( &payload[0] );
}
std::vector<uint8_t> data_out_one(decoder_one->block_size(), '\0');
std::vector<uint8_t> data_out_two(decoder_two->block_size(), '\0');
decoder_one->copy_symbols(sak::storage(data_out_one));
decoder_two->copy_symbols(sak::storage(data_out_two));
EXPECT_TRUE(std::equal(data_out_one.begin(),
data_out_one.end(),
data_in.begin()));
EXPECT_TRUE(std::equal(data_out_two.begin(),
data_out_two.end(),
data_in.begin()));
}
// Tests that encoding and decoding a file withe the file encoder
// works.
TEST(TestFileEncoder, test_file_encoder)
{
{
std::string encode_filename = "encode-file";
std::string decode_filename = "decode-file";
// Write a test file
std::ofstream encode_file;
encode_file.open (encode_filename, std::ios::binary);
uint32_t size = 500;
for(uint32_t i = 0; i < size; ++i)
{
char c = rand() % 255;
encode_file.write(&c, 1);
}
encode_file.close();
typedef kodo::full_rlnc_encoder<fifi::binary>
encoder_t;
typedef kodo::full_rlnc_decoder<fifi::binary>
decoder_t;
typedef kodo::file_encoder<encoder_t>
file_encoder_t;
typedef kodo::object_decoder<decoder_t>
object_decoder_t;
uint32_t max_symbols = 10;
uint32_t max_symbol_size = 10;
file_encoder_t::factory encoder_factory(
max_symbols, max_symbol_size);
file_encoder_t file_encoder(encoder_factory, encode_filename);
object_decoder_t::factory decoder_factory(
max_symbols, max_symbol_size);
object_decoder_t object_decoder(decoder_factory, size);
EXPECT_EQ(object_decoder.decoders(), file_encoder.encoders());
// Open the decode file
std::ofstream decode_file;
decode_file.open (decode_filename, std::ios::binary);
for(uint32_t i = 0; i < file_encoder.encoders(); ++i)
{
auto encoder = file_encoder.build(i);
auto decoder = object_decoder.build(i);
EXPECT_EQ(encoder->symbols(), decoder->symbols());
EXPECT_EQ(encoder->symbol_size(), decoder->symbol_size());
EXPECT_EQ(encoder->bytes_used(), decoder->bytes_used());
// Set the encoder non-systematic
if(kodo::is_systematic_encoder(encoder))
kodo::set_systematic_off(encoder);
std::vector<uint8_t> payload(encoder->payload_size());
while( !decoder->is_complete() )
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// Pass that packet to the decoder
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
decode_file.write(
reinterpret_cast<char*>(&data_out[0]), decoder->bytes_used());
}
decode_file.close();
}
}
