// Given a single bucket definition, create multiple buckets void BucketStore::createBucketsFromBucket(pStoreConf configuration, pStoreConf bucket_conf) { string error_msg, bucket_subdir, type, path, failure_bucket; bool needs_bucket_subdir = false; unsigned long bucket_offset = 0; pStoreConf tmp; // check for extra bucket definitions if (configuration->getStore("bucket0", tmp) || configuration->getStore("bucket1", tmp)) { error_msg = "bucket store has too many buckets defined"; goto handle_error; } bucket_conf->getString("type", type); if (type != "file" && type != "thriftfile") { error_msg = "store contained in a bucket store must have a type of "; error_msg += "either file or thriftfile if not defined explicitely"; goto handle_error; } needs_bucket_subdir = true; if (!configuration->getString("bucket_subdir", bucket_subdir)) { error_msg = "bucketizer containing file stores must have a bucket_subdir"; goto handle_error; } if (!bucket_conf->getString("file_path", path)) { error_msg = "file store contained by bucketizer must have a file_path"; goto handle_error; } // set starting bucket number if specified configuration->getUnsigned("bucket_offset", bucket_offset); // check if failure bucket was given a different name configuration->getString("failure_bucket", failure_bucket); // We actually create numBuckets + 1 stores. Messages are normally // hashed into buckets 1 through numBuckets, and messages that can't // be hashed are put in bucket 0. for (unsigned int i = 0; i <= numBuckets; ++i) { shared_ptr<Store> newstore = createStore(type, categoryHandled, false, multiCategory); if (!newstore) { error_msg = "can't create store of type: "; error_msg += type; goto handle_error; } // For file/thrift file buckets, create unique filepath for each bucket if (needs_bucket_subdir) { if (i == 0 && !failure_bucket.empty()) { bucket_conf->setString("file_path", path + '/' + failure_bucket); } else { // the bucket number is appended to the file path unsigned int bucket_id = i + bucket_offset; ostringstream oss; oss << path << '/' << bucket_subdir << setw(3) << setfill('0') << bucket_id; bucket_conf->setString("file_path", oss.str()); } } buckets.push_back(newstore); newstore->configure(bucket_conf); } return; handle_error: setStatus(error_msg); LOG_OPER("[%s] Bad config - %s", categoryHandled.c_str(), error_msg.c_str()); numBuckets = 0; buckets.clear(); }