void Map::optimize(const fs::path& directory, const fs::path& target,
const Options& options) {
const auto descriptor = internal::Descriptor::readFromDirectory(directory);
checkDescriptor(descriptor, directory);
fs::path prefix = directory / getPartitionPrefix(0);
const Stats stats = internal::Partition::stats(prefix);
Options new_options = options;
new_options.error_if_exists = true;
new_options.create_if_missing = true;
if (options.block_size == 0) {
new_options.block_size = stats.block_size;
}
if (options.num_partitions == 0) {
new_options.num_partitions = descriptor.num_partitions;
}
Map new_map(target, new_options);
for (int i = 0; i < descriptor.num_partitions; i++) {
prefix = directory / getPartitionPrefix(i);
if (options.verbose) {
mt::log() << "Optimizing partition " << (i + 1) << " of "
<< descriptor.num_partitions << std::endl;
}
if (options.compare) {
Arena arena;
std::vector<Slice> values;
internal::Partition::forEachEntry(
prefix, [&new_map, &arena, &options, &values](const Slice& key,
Iterator* iter) {
arena.deallocateAll();
values.reserve(iter->available());
values.clear();
while (iter->hasNext()) {
values.push_back(iter->next().makeCopy(&arena));
}
std::sort(values.begin(), values.end(), options.compare);
for (const auto& value : values) {
new_map.put(key, value);
}
});
} else {
internal::Partition::forEachEntry(
prefix, [&new_map](const Slice& key, Iterator* iter) {
while (iter->hasNext()) {
new_map.put(key, iter->next());
}
});
}
}
}
Map::Map(const fs::path& directory, const Options& options)
: dlock_(directory.string()) {
checkOptions(options);
Options partition_options;
partition_options.readonly = options.readonly;
partition_options.block_size = options.block_size;
internal::Descriptor descriptor;
if (internal::Descriptor::tryReadFromDirectory(directory, &descriptor)) {
checkDescriptor(descriptor, directory);
mt::Check::isFalse(options.error_if_exists, "Map in %s already exists",
fs::absolute(directory).c_str());
partitions_.resize(descriptor.num_partitions);
} else {
mt::Check::isTrue(options.create_if_missing, "Map in %s does not exist",
fs::absolute(directory).c_str());
partitions_.resize(mt::nextPrime(options.num_partitions));
descriptor.map_type = internal::Descriptor::TYPE_MAP;
descriptor.num_partitions = partitions_.size();
descriptor.writeToDirectory(directory);
}
for (size_t i = 0; i != partitions_.size(); i++) {
const fs::path prefix = directory / getPartitionPrefix(i);
partitions_[i].reset(new internal::Partition(prefix, partition_options));
}
}
std::vector<Stats> Map::stats(const fs::path& directory) {
internal::DirectoryLock lock(directory.string());
const auto descriptor = internal::Descriptor::readFromDirectory(directory);
checkDescriptor(descriptor, directory);
std::vector<Stats> stats;
for (int i = 0; i < descriptor.num_partitions; i++) {
const fs::path prefix = directory / getPartitionPrefix(i);
stats.push_back(internal::Partition::stats(prefix));
}
return stats;
}
// Required interface:
// void process(const boost::filesystem::path& partition_prefix,
// const internal::Partition::Options& partition_options,
// size_t partition_index, size_t num_partitions);
void forEachPartition(const boost::filesystem::path& directory,
Procedure process) {
mt::DirectoryLockGuard lock(directory, getNameOfLockFile());
const auto id = Map::Id::readFromDirectory(directory);
Version::checkCompatibility(id.major_version, id.minor_version);
internal::Partition::Options partition_options;
partition_options.block_size = id.block_size;
partition_options.readonly = true;
for (size_t i = 0; i != id.num_partitions; ++i) {
const auto partition_prefix = directory / getPartitionPrefix(i);
process(partition_prefix, partition_options, i, id.num_partitions);
}
}
void Map::exportToBase64(const fs::path& directory, const fs::path& target,
const Options& options) {
internal::DirectoryLock lock(directory.string());
const auto descriptor = internal::Descriptor::readFromDirectory(directory);
checkDescriptor(descriptor, directory);
internal::TsvFileWriter writer(target);
for (int i = 0; i < descriptor.num_partitions; i++) {
const fs::path prefix = directory / getPartitionPrefix(i);
if (options.verbose) {
mt::log() << "Exporting partition " << (i + 1) << " of "
<< descriptor.num_partitions << std::endl;
}
if (options.compare) {
Arena arena;
std::vector<Slice> values;
internal::Partition::forEachEntry(
prefix, [&writer, &arena, &options, &values](const Slice& key,
Iterator* iter) {
arena.deallocateAll();
values.reserve(iter->available());
values.clear();
while (iter->hasNext()) {
values.push_back(iter->next().makeCopy(&arena));
}
std::sort(values.begin(), values.end(), options.compare);
auto range_iter = makeRangeIterator(values.begin(), values.end());
writer.write(key, &range_iter);
});
} else {
internal::Partition::forEachEntry(
prefix, [&writer](const Slice& key, Iterator* iter) {
writer.write(key, iter);
});
}
}
}
