void sort(const SortParameters& parameters) {
std::uint32_t chunk_no = split_into_chunks(parameters.input_filename, parameters.chunk_size);
const std::uint32_t initial_chunks_count = chunk_no;
std::cout << "Split into " << initial_chunks_count << " chunks." << std::endl;
ThreadPool pool(parameters.threads_count);
std::vector<std::future<bool>> schedule;
// Add tasks for sorting initial chunks
for (std::uint32_t i = 0; i < initial_chunks_count; ++i) {
schedule.emplace_back(pool.enqueue(
[i, ¶meters]() {
sort_chunk(get_chunk_filename(i), parameters.chunk_size);
std::cout << "Chunk #" << i << " sorted" << std::endl;
return true;
}
));
}
// Add tasks for merging chunks
for (std::uint32_t i = 0; i * (parameters.no_of_ways_for_merging - 1) + 1 < initial_chunks_count; ++i) {
schedule.emplace_back(pool.enqueue(
[i, chunk_no, &schedule, ¶meters]() mutable {
std::cout << "Merging [";
std::vector<std::string> chunk_filenames;
for (std::uint32_t delta = 0; delta < parameters.no_of_ways_for_merging; ++delta) {
std::uint32_t needed_chunk_no = parameters.no_of_ways_for_merging * i + delta;
if (needed_chunk_no + 1 < schedule.size()) {
schedule[needed_chunk_no].wait();
chunk_filenames.emplace_back(get_chunk_filename(needed_chunk_no));
std::cout << needed_chunk_no << " ";
}
}
std::cout << "] into " << chunk_no << std::endl;
merge_sorted_chunks(chunk_filenames, get_chunk_filename(chunk_no));
delete_files(chunk_filenames);
return true;
}
));
++chunk_no;
}
schedule.back().get();
if (std::rename(get_chunk_filename(chunk_no - 1).c_str(), parameters.output_filename.c_str()) != 0) {
throw std::runtime_error("Cannot rename last chunk into output file");
}
}
static int chunk_end(AVFormatContext *s)
{
WebMChunkContext *wc = s->priv_data;
AVFormatContext *oc = wc->avf;
int ret;
int buffer_size;
uint8_t *buffer;
AVIOContext *pb;
char filename[MAX_FILENAME_SIZE];
if (wc->chunk_start_index == wc->chunk_index)
return 0;
// Flush the cluster in WebM muxer.
oc->oformat->write_packet(oc, NULL);
buffer_size = avio_close_dyn_buf(oc->pb, &buffer);
ret = get_chunk_filename(s, 0, filename);
if (ret < 0)
goto fail;
ret = avio_open2(&pb, filename, AVIO_FLAG_WRITE, &s->interrupt_callback, NULL);
if (ret < 0)
goto fail;
avio_write(pb, buffer, buffer_size);
ret = avio_close(pb);
if (ret < 0)
goto fail;
oc->pb = NULL;
fail:
av_free(buffer);
return (ret < 0) ? ret : 0;
}
static int webm_chunk_write_header(AVFormatContext *s)
{
WebMChunkContext *wc = s->priv_data;
AVFormatContext *oc = NULL;
int ret;
// DASH Streams can only have either one track per file.
if (s->nb_streams != 1) { return AVERROR_INVALIDDATA; }
wc->chunk_index = wc->chunk_start_index;
wc->oformat = av_guess_format("webm", s->filename, "video/webm");
if (!wc->oformat)
return AVERROR_MUXER_NOT_FOUND;
ret = chunk_mux_init(s);
if (ret < 0)
return ret;
oc = wc->avf;
ret = get_chunk_filename(s, 1, oc->filename);
if (ret < 0)
return ret;
ret = avio_open2(&oc->pb, oc->filename, AVIO_FLAG_WRITE,
&s->interrupt_callback, NULL);
if (ret < 0)
return ret;
oc->pb->seekable = 0;
ret = oc->oformat->write_header(oc);
if (ret < 0)
return ret;
avio_close(oc->pb);
return 0;
}
std::uint32_t split_into_chunks(const std::string& filename, std::uint64_t chunk_size) {
std::ifstream input_file(filename, std::ios::binary);
uint32_t chunks_count = 0;
char* buffer = new char[chunk_size];
while (!input_file.eof()) {
input_file.read(buffer, chunk_size);
auto read_length = input_file.gcount();
if (read_length > 0) {
auto chunk_filename = get_chunk_filename(chunks_count++);
std::ofstream chunk_file(chunk_filename, std::ios::binary | std::ios::trunc);
chunk_file.write(buffer, read_length);
chunk_file.close();
}
}
delete [] buffer;
return chunks_count;
}
