int exist_dtable::create(int dfd, const char * file, const params & config, dtable::iter * source, const ktable * shadow)
{
int e_dfd, r;
params base_config, dnebase_config;
const dtable_factory * base = dtable_factory::lookup(config, "base");
const dtable_factory * dnebase = dtable_factory::lookup(config, "dnebase");
if(!base || !dnebase)
return -ENOENT;
if(!config.get("base_config", &base_config, params()))
return -EINVAL;
if(!config.get("dnebase_config", &dnebase_config, params()))
return -EINVAL;
if(!source_shadow_ok(source, shadow))
return -EINVAL;
r = mkdirat(dfd, file, 0755);
if(r < 0)
return r;
e_dfd = openat(dfd, file, O_RDONLY);
if(e_dfd < 0)
goto fail_open;
/* just to be sure */
source->first();
{
dtable_skip_iter<dne_skip_test> base_source(source);
r = base->create(e_dfd, "base", base_config, &base_source, NULL);
if(r < 0)
goto fail_base;
}
source->first();
{
full_ktable full_shadow(source);
nonshadow_skip_test skip_test(shadow);
dtable_skip_iter<nonshadow_skip_test> dnebase_source(source, skip_test);
r = dnebase->create(e_dfd, "dnebase", dnebase_config, &dnebase_source, &full_shadow);
if(r < 0)
goto fail_dnebase;
}
close(e_dfd);
return 0;
fail_dnebase:
util::rm_r(e_dfd, "base");
fail_base:
close(e_dfd);
fail_open:
unlinkat(dfd, file, AT_REMOVEDIR);
return (r < 0) ? r : -1;
}
int btree_dtable::create(int dfd, const char * file, const params & config, dtable::iter * source, const ktable * shadow)
{
int bt_dfd, r;
params base_config;
dtable * base_dtable;
const dtable_factory * base = dtable_factory::lookup(config, "base");
if(!base)
return -ENOENT;
if(!config.get("base_config", &base_config, params()))
return -EINVAL;
if(!base->indexed_access(base_config))
return -ENOSYS;
if(!source_shadow_ok(source, shadow))
return -EINVAL;
r = mkdirat(dfd, file, 0755);
if(r < 0)
return r;
bt_dfd = openat(dfd, file, O_RDONLY);
if(bt_dfd < 0)
goto fail_open;
r = base->create(bt_dfd, "base", base_config, source, shadow);
if(r < 0)
goto fail_create;
base_dtable = base->open(bt_dfd, "base", base_config, NULL);
if(!base_dtable)
goto fail_reopen;
r = write_btree(bt_dfd, "btree", base_dtable);
if(r < 0)
goto fail_write;
base_dtable->destroy();
close(bt_dfd);
return 0;
fail_write:
base_dtable->destroy();
fail_reopen:
util::rm_r(bt_dfd, "base");
fail_create:
close(bt_dfd);
fail_open:
unlinkat(dfd, file, AT_REMOVEDIR);
return (r < 0) ? r : -1;
}
int exception_dtable::create(int dfd, const char * file, const params & config, dtable::iter * source, const ktable * shadow)
{
int excp_dfd, r;
memory_dtable alt_mdt;
reject_iter * handler;
params base_config, alt_config;
blob reject_value;
const dtable_factory * base = dtable_factory::lookup(config, "base");
const dtable_factory * alt = dtable_factory::lookup(config, "alt");
if(!base || !alt)
return -EINVAL;
if(!config.get("base_config", &base_config, params()))
return -EINVAL;
if(!config.get("alt_config", &alt_config, params()))
return -EINVAL;
if(!config.get_blob_or_string("reject_value", &reject_value))
return -EINVAL;
/* the reject value must exist, because nonexistent values
* can get pruned out if the shadow does not require them */
if(!reject_value.exists())
return -EINVAL;
if(!source_shadow_ok(source, shadow))
return -EINVAL;
r = mkdirat(dfd, file, 0755);
if(r < 0)
return r;
excp_dfd = openat(dfd, file, O_RDONLY);
if(excp_dfd < 0)
goto fail_open;
/* we should really save the reject_value in a meta file here */
/* we'll always be appending, but it's faster if we say false here */
r = alt_mdt.init(source->key_type(), false, true);
if(r < 0)
goto fail_mdt;
if(source->get_blob_cmp())
alt_mdt.set_blob_cmp(source->get_blob_cmp());
handler = new reject_iter(source, &alt_mdt, reject_value);
if(!handler)
goto fail_mdt;
r = base->create(excp_dfd, "base", base_config, handler, shadow);
if(r < 0)
goto fail_base;
/* no shadow - this only has exceptions */
r = alt->create(excp_dfd, "alt", alt_config, &alt_mdt, NULL);
if(r < 0)
goto fail_alt;
delete handler;
close(excp_dfd);
return 0;
fail_alt:
util::rm_r(excp_dfd, "base");
fail_base:
delete handler;
fail_mdt:
close(excp_dfd);
fail_open:
unlinkat(dfd, file, AT_REMOVEDIR);
return -1;
}
int simple_dtable::create(int dfd, const char * file, const params & config, dtable::iter * source, const ktable * shadow)
{
std::vector<istr> strings;
std::vector<blob> blobs;
dtype::ctype key_type = source->key_type();
const blob_comparator * blob_cmp = source->get_blob_cmp();
size_t key_count = 0, max_data_size = 0, total_data_size = 0;
uint32_t max_key = 0;
dtable_header header;
int r, size;
rwfile out;
if(!source_shadow_ok(source, shadow))
return -EINVAL;
/* just to be sure */
source->first();
while(source->valid())
{
dtype key = source->key();
metablob meta = source->meta();
source->next();
if(!meta.exists())
/* omit non-existent entries no longer needed */
if(!shadow || !shadow->contains(key))
continue;
key_count++;
assert(key.type == key_type);
switch(key.type)
{
case dtype::UINT32:
if(key.u32 > max_key)
max_key = key.u32;
break;
case dtype::DOUBLE:
/* nothing to do */
break;
case dtype::STRING:
strings.push_back(key.str);
break;
case dtype::BLOB:
blobs.push_back(key.blb);
break;
}
if(meta.size() > max_data_size)
max_data_size = meta.size();
total_data_size += meta.size();
}
/* now write the file */
header.magic = SDTABLE_MAGIC;
header.version = SDTABLE_VERSION;
header.key_count = key_count;
switch(key_type)
{
case dtype::UINT32:
header.key_type = 1;
header.key_size = util::byte_size(max_key);
break;
case dtype::DOUBLE:
header.key_type = 2;
header.key_size = sizeof(double);
break;
case dtype::STRING:
header.key_type = 3;
header.key_size = util::byte_size(strings.size() - 1);
break;
case dtype::BLOB:
header.key_type = 4;
header.key_size = util::byte_size(blobs.size() - 1);
break;
}
/* we reserve size 0 for non-existent entries, so add 1 */
header.length_size = util::byte_size(max_data_size + 1);
header.offset_size = util::byte_size(total_data_size);
size = header.key_size + header.length_size + header.offset_size;
r = out.create(dfd, file);
if(r < 0)
return r;
r = out.append(&header);
if(r < 0)
goto fail_unlink;
if(key_type == dtype::BLOB)
{
uint32_t length = blob_cmp ? strlen(blob_cmp->name) : 0;
out.append(&length);
if(length)
out.append(blob_cmp->name);
}
if(key_type == dtype::STRING)
{
r = stringtbl::create(&out, strings);
if(r < 0)
goto fail_unlink;
}
else if(key_type == dtype::BLOB)
{
r = stringtbl::create(&out, blobs);
if(r < 0)
goto fail_unlink;
}
/* now the key array */
max_key = 0;
total_data_size = 0;
source->first();
while(source->valid())
{
int i = 0;
uint8_t bytes[size];
dtype key = source->key();
metablob meta = source->meta();
source->next();
if(!meta.exists())
/* omit non-existent entries no longer needed */
if(!shadow || !shadow->contains(key))
continue;
switch(key.type)
{
case dtype::UINT32:
util::layout_bytes(bytes, &i, key.u32, header.key_size);
break;
case dtype::DOUBLE:
util::memcpy(bytes, &key.dbl, sizeof(double));
i += sizeof(double);
break;
case dtype::STRING:
/* no need to locate the string; it's the next one */
util::layout_bytes(bytes, &i, max_key, header.key_size);
max_key++;
break;
case dtype::BLOB:
/* no need to locate the blob; it's the next one */
util::layout_bytes(bytes, &i, max_key, header.key_size);
max_key++;
break;
}
util::layout_bytes(bytes, &i, meta.exists() ? meta.size() + 1 : 0, header.length_size);
util::layout_bytes(bytes, &i, total_data_size, header.offset_size);
r = out.append(bytes, i);
if(r != i)
goto fail_unlink;
total_data_size += meta.size();
}
/* and the data itself */
source->first();
while(source->valid())
{
blob value = source->value();
source->next();
/* nonexistent blobs have size 0 */
if(!value.size())
continue;
r = out.append(value);
if(r < 0)
goto fail_unlink;
}
r = out.close();
if(r < 0)
goto fail_unlink;
return 0;
fail_unlink:
out.close();
unlinkat(dfd, file, 0);
return (r < 0) ? r : -1;
}
int deltaint_dtable::create(int dfd, const char * file, const params & config, dtable::iter * source, const ktable * shadow)
{
int r, di_dfd, skip;
rev_iter_delta * rev;
rev_iter_ref * ref;
params base_config, ref_config;
const dtable_factory * base = dtable_factory::lookup(config, "base");
const dtable_factory * reference = dtable_factory::lookup(config, "ref");
if(!base || !reference)
return -ENOENT;
if(!config.get("base_config", &base_config, params()))
return -EINVAL;
if(!config.get("ref_config", &ref_config, params()))
return -EINVAL;
if(!config.get("skip", &skip, 0) || skip < 2)
return -EINVAL;
if(!source_shadow_ok(source, shadow))
return -EINVAL;
r = mkdirat(dfd, file, 0755);
if(r < 0)
return r;
di_dfd = openat(dfd, file, O_RDONLY);
if(di_dfd < 0)
goto fail_open;
rev = new rev_iter_delta(source);
if(!rev)
{
r = -ENOMEM;
goto fail_iter_1;
}
r = base->create(di_dfd, "base", base_config, rev, shadow);
if(r < 0)
goto fail_create_1;
if(rev->failed)
{
r = -ENOSYS;
goto fail_iter_2;
}
ref = new rev_iter_ref(source, skip);
if(!ref)
{
r = -ENOMEM;
goto fail_iter_2;
}
r = reference->create(di_dfd, "ref", ref_config, ref, NULL);
if(r < 0)
goto fail_create_2;
delete ref;
delete rev;
close(di_dfd);
return r;
fail_create_2:
delete ref;
fail_iter_2:
util::rm_r(di_dfd, "base");
fail_create_1:
delete rev;
fail_iter_1:
close(di_dfd);
fail_open:
unlinkat(dfd, file, AT_REMOVEDIR);
return (r < 0) ? r : -1;
}