/** * Buckets in a bucket store can be defined explicitly or implicitly: * * #Explicitly * <store> * type=bucket * num_buckets=2 * bucket_type=key_hash * * <bucket0> * ... * </bucket0> * * <bucket1> * ... * </bucket1> * * <bucket2> * ... * </bucket2> * </store> * * #Implicitly * <store> * type=bucket * num_buckets=2 * bucket_type=key_hash * * <bucket> * ... * </bucket> * </store> */ void BucketStore::configure(pStoreConf configuration) { string error_msg, bucketizer_str, remove_key_str; unsigned long delim_long = 0; pStoreConf bucket_conf; //set this to true for bucket types that have a delimiter bool need_delimiter = false; configuration->getString("bucket_type", bucketizer_str); // Figure out th bucket type from the bucketizer string if (0 == bucketizer_str.compare("context_log")) { bucketType = context_log; } else if (0 == bucketizer_str.compare("random")) { bucketType = random; } else if (0 == bucketizer_str.compare("key_hash")) { bucketType = key_hash; need_delimiter = true; } else if (0 == bucketizer_str.compare("key_modulo")) { bucketType = key_modulo; need_delimiter = true; } else if (0 == bucketizer_str.compare("key_range")) { bucketType = key_range; need_delimiter = true; configuration->getUnsigned("bucket_range", bucketRange); if (bucketRange == 0) { LOG_OPER("[%s] config warning - bucket_range is 0", categoryHandled.c_str()); } } // This is either a key_hash or key_modulo, not context log, figure out the delimiter and store it if (need_delimiter) { configuration->getUnsigned("delimiter", delim_long); if (delim_long > 255) { LOG_OPER("[%s] config warning - delimiter is too large to fit in a char, using default", categoryHandled.c_str()); delimiter = DEFAULT_BUCKETSTORE_DELIMITER; } else if (delim_long == 0) { LOG_OPER("[%s] config warning - delimiter is zero, using default", categoryHandled.c_str()); delimiter = DEFAULT_BUCKETSTORE_DELIMITER; } else { delimiter = (char)delim_long; } } // Optionally remove the key and delimiter of each message before bucketizing configuration->getString("remove_key", remove_key_str); if (remove_key_str == "yes") { removeKey = true; if (bucketType == context_log) { error_msg = "Bad config - bucketizer store of type context_log do not support remove_key"; goto handle_error; } } if (!configuration->getUnsigned("num_buckets", numBuckets)) { error_msg = "Bad config - bucket store must have num_buckets"; goto handle_error; } // Buckets can be defined explicitely or by specifying a single "bucket" if (configuration->getStore("bucket", bucket_conf)) { createBucketsFromBucket(configuration, bucket_conf); } else { createBuckets(configuration); } return; handle_error: setStatus(error_msg); LOG_OPER("[%s] %s", categoryHandled.c_str(), error_msg.c_str()); numBuckets = 0; buckets.clear(); }
// Given a single bucket definition, create multiple buckets void BucketStore::createBucketsFromBucket(pStoreConf configuration, pStoreConf bucket_conf) { string error_msg, bucket_subdir, type, path, failure_bucket; bool needs_bucket_subdir = false; unsigned long bucket_offset = 0; pStoreConf tmp; // check for extra bucket definitions if (configuration->getStore("bucket0", tmp) || configuration->getStore("bucket1", tmp)) { error_msg = "bucket store has too many buckets defined"; goto handle_error; } bucket_conf->getString("type", type); if (type != "file" && type != "thriftfile") { error_msg = "store contained in a bucket store must have a type of "; error_msg += "either file or thriftfile if not defined explicitely"; goto handle_error; } needs_bucket_subdir = true; if (!configuration->getString("bucket_subdir", bucket_subdir)) { error_msg = "bucketizer containing file stores must have a bucket_subdir"; goto handle_error; } if (!bucket_conf->getString("file_path", path)) { error_msg = "file store contained by bucketizer must have a file_path"; goto handle_error; } // set starting bucket number if specified configuration->getUnsigned("bucket_offset", bucket_offset); // check if failure bucket was given a different name configuration->getString("failure_bucket", failure_bucket); // We actually create numBuckets + 1 stores. Messages are normally // hashed into buckets 1 through numBuckets, and messages that can't // be hashed are put in bucket 0. for (unsigned int i = 0; i <= numBuckets; ++i) { shared_ptr<Store> newstore = createStore(type, categoryHandled, false, multiCategory); if (!newstore) { error_msg = "can't create store of type: "; error_msg += type; goto handle_error; } // For file/thrift file buckets, create unique filepath for each bucket if (needs_bucket_subdir) { if (i == 0 && !failure_bucket.empty()) { bucket_conf->setString("file_path", path + '/' + failure_bucket); } else { // the bucket number is appended to the file path unsigned int bucket_id = i + bucket_offset; ostringstream oss; oss << path << '/' << bucket_subdir << setw(3) << setfill('0') << bucket_id; bucket_conf->setString("file_path", oss.str()); } } buckets.push_back(newstore); newstore->configure(bucket_conf); } return; handle_error: setStatus(error_msg); LOG_OPER("[%s] Bad config - %s", categoryHandled.c_str(), error_msg.c_str()); numBuckets = 0; buckets.clear(); }
// Checks for a bucket definition for every bucket from 0 to numBuckets // and configures each bucket void BucketStore::createBuckets(pStoreConf configuration) { string error_msg, tmp_string; pStoreConf tmp; unsigned long i; if (configuration->getString("bucket_subdir", tmp_string)) { error_msg = "cannot have bucket_subdir when defining multiple buckets"; goto handle_error; } if (configuration->getString("bucket_offset", tmp_string)) { error_msg = "cannot have bucket_offset when defining multiple buckets"; goto handle_error; } if (configuration->getString("failure_bucket", tmp_string)) { error_msg = "cannot have failure_bucket when defining multiple buckets"; goto handle_error; } // Configure stores named 'bucket0, bucket1, bucket2, ... bucket{numBuckets} for (i = 0; i <= numBuckets; i++) { pStoreConf bucket_conf; string type, bucket_name; stringstream ss; ss << "bucket" << i; bucket_name = ss.str(); if (!configuration->getStore(bucket_name, bucket_conf)) { error_msg = "could not find bucket definition for " + bucket_name; goto handle_error; } if (!bucket_conf->getString("type", type)) { error_msg = "store contained in a bucket store must have a type"; goto handle_error; } shared_ptr<Store> bucket = createStore(type, categoryHandled, false, multiCategory); buckets.push_back(bucket); bucket->configure(bucket_conf); } // Check if an extra bucket is defined if (configuration->getStore("bucket" + (numBuckets + 1), tmp)) { error_msg = "bucket store has too many buckets defined"; goto handle_error; } return; handle_error: setStatus(error_msg); LOG_OPER("[%s] Bad config - %s", categoryHandled.c_str(), error_msg.c_str()); numBuckets = 0; buckets.clear(); }