StoragePtr TableFunctionShardByHash::execute(ASTPtr ast_function, Context & context) const
{
ASTs & args_func = typeid_cast<ASTFunction &>(*ast_function).children;
const char * err = "Table function 'shardByHash' requires 4 parameters: "
"cluster name, key string to hash, name of remote database, name of remote table.";
if (args_func.size() != 1)
throw Exception(err, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
ASTs & args = typeid_cast<ASTExpressionList &>(*args_func.at(0)).children;
if (args.size() != 4)
throw Exception(err, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
String cluster_name;
String key;
String remote_database;
String remote_table;
auto getStringLiteral = [](const IAST & node, const char * description)
{
const ASTLiteral * lit = typeid_cast<const ASTLiteral *>(&node);
if (!lit)
throw Exception(description + String(" must be string literal (in single quotes)."), ErrorCodes::BAD_ARGUMENTS);
if (lit->value.getType() != Field::Types::String)
throw Exception(description + String(" must be string literal (in single quotes)."), ErrorCodes::BAD_ARGUMENTS);
return safeGet<const String &>(lit->value);
};
cluster_name = getClusterName(*args[0]);
key = getStringLiteral(*args[1], "Key to hash");
args[2] = evaluateConstantExpressionOrIdentidierAsLiteral(args[2], context);
args[3] = evaluateConstantExpressionOrIdentidierAsLiteral(args[3], context);
remote_database = static_cast<const ASTLiteral &>(*args[2]).value.safeGet<String>();
remote_table = static_cast<const ASTLiteral &>(*args[3]).value.safeGet<String>();
/// Аналогично другим TableFunctions.
for (auto & arg : args)
if (ASTIdentifier * id = typeid_cast<ASTIdentifier *>(arg.get()))
id->kind = ASTIdentifier::Table;
const Cluster & cluster = context.getCluster(cluster_name);
size_t shard_index = sipHash64(key) % cluster.getShardCount();
std::shared_ptr<Cluster> shard(cluster.getClusterWithSingleShard(shard_index).release());
return StorageDistributed::create(
getName(),
std::make_shared<NamesAndTypesList>(getStructureOfRemoteTable(*shard, remote_database, remote_table, context)),
remote_database,
remote_table,
shard,
context);
}
void ZkManager::notifyHandle(std::string path) {
if( path == "" ){
std::set<std::string> failClusterSet = notifyChangeAll();
if( failClusterSet.size() > 0 ) {
addFailCluster( failClusterSet );
}
}else {
std::string clusterName = getClusterName(path);
if( notifyChange( clusterName ) == false ) {
addFailCluster( clusterName );
}
}
}
int addBinPartner(mapper_t map, char *fileName) {
int res = 0;
int size, i;
char *buff;
struct bin_partner *partnerArr;
int rangeNum;
char *clusterName = NULL;
cluster_entry_t *cEnt;
mapper_cluster_info_t ci;
mapper_range_info_t ri;
buff = getMapSourceBuff(fileName, 0, INPUT_FILE_BIN, &size);
if(!buff) {
debug_lr(MAP_DEBUG, "Error reading file\n");
goto out;
}
if(size == 0 || (size % sizeof(struct bin_partner)) != 0) {
debug_lr(MAP_DEBUG, "Error number of records is not integer (%d %d)\n",
size, sizeof(struct bin_partner));
goto out;
}
clusterName = getClusterName(fileName);
partnerArr = (struct bin_partner *) buff;
rangeNum = size / sizeof(struct bin_partner);
// Adding the cluster using the first range
ci.ci_prio = partnerArr[0].priority;
ci.ci_cango = !partnerArr[0].dontgo;
ci.ci_cantake = !partnerArr[0].donttake;
ci.ci_canexpend = 0;
ci.ci_desc = NULL;
clusterName = getClusterName(fileName);
if(!mapper_addCluster(map, -1, clusterName, &ci)) {
debug_lr(MAP_DEBUG, "Failed to add cluster %s\n", clusterName);
goto out;
}
// Checking that the cluster is not already there
cEnt = get_cluster_entry_by_name(map, clusterName);
if(!cEnt) {
debug_lr(MAP_DEBUG, "Just added cluster is not there\n");
goto out;
}
// Adding all the ranges
int base = 1;
for(i = 0 ; i < rangeNum ; i++) {
ri.ri_core = 0;
ri.ri_participate = 1;
ri.ri_proximate = partnerArr[i].proximate;
struct in_addr addr;
addr.s_addr = htonl(partnerArr[i].ip);
//printf("Got range %s %d\n", inet_ntoa(addr), partnerArr[i].n);
// If the range is valid adding it to the cluster (the single cluster we have)
if(mapper_addClusterAddrRange(map, cEnt, base, &addr, partnerArr[i].n, &ri))
{
base += partnerArr[i].n;
}
else {
goto out;
}
}
res = 1;
out:
if(buff)
free(buff);
return res;
}
int addTextPartner(mapper_t map, char *fileName) {
int res = 0;
int size;
int linenum=0;
char *buffCurrPos = NULL;
char *buff = NULL;
char *clusterName = NULL;
cluster_entry_t *cEnt = NULL;
mapper_cluster_info_t ci;
mapper_range_info_t ri;
int lineBuffSize = 200;
char lineBuff[200];
char host_name[255];
cluster_info_init(&ci);
buff = getMapSourceBuff(fileName, 0, INPUT_FILE, &size);
if(!buff) {
debug_lr(MAP_DEBUG, "Error reading file\n");
goto out;
}
buffCurrPos = (char*) buff;
if(!buffCurrPos)
goto out;
// Reading the cluster commet/description
if(lineBuffSize > size)
lineBuffSize = size;
buffCurrPos = buff_get_line(lineBuff, lineBuffSize, buffCurrPos);
if(!buffCurrPos) {
debug_lr(MAP_DEBUG, "First line is not ok\n");
goto out;
}
linenum++;
debug_lg(MAP_DEBUG, "Processing line: %s\n", lineBuff);
ci.ci_desc = strdup(lineBuff);
// Reading the cluster prio, dont-take dontgo...
buffCurrPos = buff_get_line(lineBuff, lineBuffSize, buffCurrPos);
if(!buffCurrPos) {
debug_lr(MAP_DEBUG, "Second line is ok\n");
goto out;
}
linenum++;
debug_lg(MAP_DEBUG, "Processing line: %s\n", lineBuff);
int cango, cantake, canexpend;
if(sscanf(lineBuff, "%d %d %d %d",
&ci.ci_prio, &cango, &cantake, &canexpend) != 4)
{
debug_lr(MAP_DEBUG, "Second line is not in correct format\n");
goto out;
}
ci.ci_cango = (char)cango;
ci.ci_cantake = (char)cantake;
ci.ci_canexpend = (char)canexpend;
clusterName = getClusterName(fileName);
if(!mapper_addCluster(map, -1, clusterName, &ci)) {
debug_lr(MAP_DEBUG, "Failed to add cluster %s\n", clusterName);
goto out;
}
free(ci.ci_desc);
ci.ci_desc = NULL;
// Checking that the cluster is not already there
cEnt = get_cluster_entry_by_name(map, clusterName);
if(!cEnt) {
debug_lr(MAP_DEBUG, "Just added cluster is not there\n");
goto out;
}
// Reading the ranges
int base = 1;
while( buffCurrPos && !(*buffCurrPos == '\0') )
{
linenum++;
buffCurrPos = buff_get_line(lineBuff, lineBuffSize, buffCurrPos);
debug_lg(MAP_DEBUG, "Processing line: %s\n", lineBuff);
// Skeeing comments in map
if(lineBuff[0] == '#')
{
debug_lg(MAP_DEBUG, "\t\tcomment\n");
continue;
}
// Skeeping empty lines
if(strlen(lineBuff) == 0)
continue;
int count, core, participate, proximate;
if (sscanf(lineBuff, "%s %d %d %d %d",
host_name, &count, &core, &participate, &proximate) == 5) {
debug_lb(MAP_DEBUG, "Found partner range entry %s %d\n",
host_name, count);
ri.ri_core = (char) core;
ri.ri_participate = (char) participate;
ri.ri_proximate = (char) proximate;
// If the range is valid adding it to the cluster (the
// single cluster we have)
// Doing the initialization part only in the first time
// the map become old
if(mapper_addClusterHostRange(map, cEnt, base, host_name, count, &ri))
{
base += count;
}
else {
goto out;
}
}
else {
lineBuff[strlen(lineBuff)] = '\0';
debug_lr(MAP_DEBUG, "Found bad line %s\n", lineBuff);
sprintf(map->errorMsg, "Bad line <%s>\n", lineBuff);
goto out;
}
}
res = 1;
out:
if(buff)
free(buff);
cluster_info_free(&ci);
return res;
}
std::string DynomiteConfig::generateConfigFile() {
std::string localIp = getIpAddress();
LOG( INFO ) << "Get local ip address [" << localIp << "].";
std::string filename = getRandomTempFile();
LOG( INFO ) << "Generate temporary configuration at [" << filename << "].";
int token = std::rand();
LOG( INFO ) << "Generate dynomite token [" << token << "].";
std::string serviceName = getClusterName();
LOG( INFO ) << "Get dynomite service name [" << serviceName << "].";
int port = getDynomitePort();
LOG( INFO ) << "Get dynomite port [" << port << "].";
consul_datacenter dc;
int ret = client->getDefaultDatacenter( dc );
if( ret != 0 ) {
LOG( ERROR ) << "Error in getting data center information from consul:" << ret;
exit( ret );
}
std::stringstream ss;
ss << localIp << ":" << port << ":default-rack-" << toLowerCase( dc.name )
<< ":" << dc.name << ":" << token;
std::string tagId = ss.str();
LOG( INFO ) << "Generate tag id [" << tagId << "].";
consul_service dynomiteService;
dynomiteService.service_id = serviceName + "_" + localIp;
dynomiteService.service_name = serviceName;
dynomiteService.service_address = localIp;
dynomiteService.service_port = port;
std::vector<consul_service> services;
client->getService( serviceName, services );
std::string seed = "";
if( !services.empty() ) {
std::vector<consul_service>::iterator it =
std::find_if( services.begin(), services.end(),
[]( consul_service cs ) -> bool {
return cs.tags.find( "seed" ) != cs.tags.end();
} );
if( it != services.end() ){
std::set<std::string>::iterator tagIt =
std::find_if( it->tags.begin(), it->tags.end(),
[]( std::string tag ) -> bool {
return tag != "seed";
} );
seed = *tagIt;
}
}
std::ofstream ofstream;
ofstream.open( filename );
ofstream << getClusterName() << ":" << std::endl;
ofstream << " datacenter: " << dc.name << std::endl;
ofstream << " rack: default-rack-" << toLowerCase( dc.name ) << std::endl;
ofstream << " dyn_listen: 0.0.0.0:" << port << std::endl;
ofstream << " listen: 0.0.0.0:" << ( port + 1 ) << std::endl;
ofstream << " servers:" << std::endl;
ofstream << " - 127.0.0.1:6379:1" << std::endl;
ofstream << " tokens: '" << token << "'" << std::endl;
ofstream << " redis: true" << std::endl;
ofstream << " secure_server_option: datacenter" << std::endl;
ofstream << " pem_key_file: /var/dynomite/dynomite.pem" << std::endl;
if( seed.empty() ) {
LOG( INFO ) << "No seed server, start as seed server.";
dynomiteService.tags.insert( "seed" );
dynomiteService.tags.insert( tagId );
} else {
LOG( INFO ) << "Find seed [" << seed << "]";
ofstream << " dyn_seeds:" << std::endl;
ofstream << " - " << seed << std::endl;
dynomiteService.tags.insert( tagId );
}
ofstream.close();
ret = client->registerService( dynomiteService );
if( ret != 0 ){
LOG( ERROR ) << "Error to register dynomite service, return " << ret;
exit( ret );
}
return filename;
}