/**
* Buckets in a bucket store can be defined explicitly or implicitly:
*
* #Explicitly
* <store>
* type=bucket
* num_buckets=2
* bucket_type=key_hash
*
* <bucket0>
* ...
* </bucket0>
*
* <bucket1>
* ...
* </bucket1>
*
* <bucket2>
* ...
* </bucket2>
* </store>
*
* #Implicitly
* <store>
* type=bucket
* num_buckets=2
* bucket_type=key_hash
*
* <bucket>
* ...
* </bucket>
* </store>
*/
void BucketStore::configure(pStoreConf configuration) {
string error_msg, bucketizer_str, remove_key_str;
unsigned long delim_long = 0;
pStoreConf bucket_conf;
//set this to true for bucket types that have a delimiter
bool need_delimiter = false;
configuration->getString("bucket_type", bucketizer_str);
// Figure out th bucket type from the bucketizer string
if (0 == bucketizer_str.compare("context_log")) {
bucketType = context_log;
} else if (0 == bucketizer_str.compare("random")) {
bucketType = random;
} else if (0 == bucketizer_str.compare("key_hash")) {
bucketType = key_hash;
need_delimiter = true;
} else if (0 == bucketizer_str.compare("key_modulo")) {
bucketType = key_modulo;
need_delimiter = true;
} else if (0 == bucketizer_str.compare("key_range")) {
bucketType = key_range;
need_delimiter = true;
configuration->getUnsigned("bucket_range", bucketRange);
if (bucketRange == 0) {
LOG_OPER("[%s] config warning - bucket_range is 0",
categoryHandled.c_str());
}
}
// This is either a key_hash or key_modulo, not context log, figure out the delimiter and store it
if (need_delimiter) {
configuration->getUnsigned("delimiter", delim_long);
if (delim_long > 255) {
LOG_OPER("[%s] config warning - delimiter is too large to fit in a char, using default", categoryHandled.c_str());
delimiter = DEFAULT_BUCKETSTORE_DELIMITER;
} else if (delim_long == 0) {
LOG_OPER("[%s] config warning - delimiter is zero, using default", categoryHandled.c_str());
delimiter = DEFAULT_BUCKETSTORE_DELIMITER;
} else {
delimiter = (char)delim_long;
}
}
// Optionally remove the key and delimiter of each message before bucketizing
configuration->getString("remove_key", remove_key_str);
if (remove_key_str == "yes") {
removeKey = true;
if (bucketType == context_log) {
error_msg =
"Bad config - bucketizer store of type context_log do not support remove_key";
goto handle_error;
}
}
if (!configuration->getUnsigned("num_buckets", numBuckets)) {
error_msg = "Bad config - bucket store must have num_buckets";
goto handle_error;
}
// Buckets can be defined explicitely or by specifying a single "bucket"
if (configuration->getStore("bucket", bucket_conf)) {
createBucketsFromBucket(configuration, bucket_conf);
} else {
createBuckets(configuration);
}
return;
handle_error:
setStatus(error_msg);
LOG_OPER("[%s] %s", categoryHandled.c_str(), error_msg.c_str());
numBuckets = 0;
buckets.clear();
}
// Given a single bucket definition, create multiple buckets
void BucketStore::createBucketsFromBucket(pStoreConf configuration,
pStoreConf bucket_conf) {
string error_msg, bucket_subdir, type, path, failure_bucket;
bool needs_bucket_subdir = false;
unsigned long bucket_offset = 0;
pStoreConf tmp;
// check for extra bucket definitions
if (configuration->getStore("bucket0", tmp) ||
configuration->getStore("bucket1", tmp)) {
error_msg = "bucket store has too many buckets defined";
goto handle_error;
}
bucket_conf->getString("type", type);
if (type != "file" && type != "thriftfile") {
error_msg = "store contained in a bucket store must have a type of ";
error_msg += "either file or thriftfile if not defined explicitely";
goto handle_error;
}
needs_bucket_subdir = true;
if (!configuration->getString("bucket_subdir", bucket_subdir)) {
error_msg =
"bucketizer containing file stores must have a bucket_subdir";
goto handle_error;
}
if (!bucket_conf->getString("file_path", path)) {
error_msg =
"file store contained by bucketizer must have a file_path";
goto handle_error;
}
// set starting bucket number if specified
configuration->getUnsigned("bucket_offset", bucket_offset);
// check if failure bucket was given a different name
configuration->getString("failure_bucket", failure_bucket);
// We actually create numBuckets + 1 stores. Messages are normally
// hashed into buckets 1 through numBuckets, and messages that can't
// be hashed are put in bucket 0.
for (unsigned int i = 0; i <= numBuckets; ++i) {
shared_ptr<Store> newstore =
createStore(type, categoryHandled, false, multiCategory);
if (!newstore) {
error_msg = "can't create store of type: ";
error_msg += type;
goto handle_error;
}
// For file/thrift file buckets, create unique filepath for each bucket
if (needs_bucket_subdir) {
if (i == 0 && !failure_bucket.empty()) {
bucket_conf->setString("file_path", path + '/' + failure_bucket);
} else {
// the bucket number is appended to the file path
unsigned int bucket_id = i + bucket_offset;
ostringstream oss;
oss << path << '/' << bucket_subdir << setw(3) << setfill('0')
<< bucket_id;
bucket_conf->setString("file_path", oss.str());
}
}
buckets.push_back(newstore);
newstore->configure(bucket_conf);
}
return;
handle_error:
setStatus(error_msg);
LOG_OPER("[%s] Bad config - %s", categoryHandled.c_str(),
error_msg.c_str());
numBuckets = 0;
buckets.clear();
}
// Checks for a bucket definition for every bucket from 0 to numBuckets
// and configures each bucket
void BucketStore::createBuckets(pStoreConf configuration) {
string error_msg, tmp_string;
pStoreConf tmp;
unsigned long i;
if (configuration->getString("bucket_subdir", tmp_string)) {
error_msg =
"cannot have bucket_subdir when defining multiple buckets";
goto handle_error;
}
if (configuration->getString("bucket_offset", tmp_string)) {
error_msg =
"cannot have bucket_offset when defining multiple buckets";
goto handle_error;
}
if (configuration->getString("failure_bucket", tmp_string)) {
error_msg =
"cannot have failure_bucket when defining multiple buckets";
goto handle_error;
}
// Configure stores named 'bucket0, bucket1, bucket2, ... bucket{numBuckets}
for (i = 0; i <= numBuckets; i++) {
pStoreConf bucket_conf;
string type, bucket_name;
stringstream ss;
ss << "bucket" << i;
bucket_name = ss.str();
if (!configuration->getStore(bucket_name, bucket_conf)) {
error_msg = "could not find bucket definition for " +
bucket_name;
goto handle_error;
}
if (!bucket_conf->getString("type", type)) {
error_msg =
"store contained in a bucket store must have a type";
goto handle_error;
}
shared_ptr<Store> bucket =
createStore(type, categoryHandled, false, multiCategory);
buckets.push_back(bucket);
bucket->configure(bucket_conf);
}
// Check if an extra bucket is defined
if (configuration->getStore("bucket" + (numBuckets + 1), tmp)) {
error_msg = "bucket store has too many buckets defined";
goto handle_error;
}
return;
handle_error:
setStatus(error_msg);
LOG_OPER("[%s] Bad config - %s", categoryHandled.c_str(),
error_msg.c_str());
numBuckets = 0;
buckets.clear();
}
