StoragePtr TableFunctionShardByHash::execute(ASTPtr ast_function, Context & context) const
{
	ASTs & args_func = typeid_cast<ASTFunction &>(*ast_function).children;

	const char * err = "Table function 'shardByHash' requires 4 parameters: "
		"cluster name, key string to hash, name of remote database, name of remote table.";

	if (args_func.size() != 1)
		throw Exception(err, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);

	ASTs & args = typeid_cast<ASTExpressionList &>(*args_func.at(0)).children;

	if (args.size() != 4)
		throw Exception(err, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);

	String cluster_name;
	String key;
	String remote_database;
	String remote_table;

	auto getStringLiteral = [](const IAST & node, const char * description)
	{
		const ASTLiteral * lit = typeid_cast<const ASTLiteral *>(&node);
		if (!lit)
			throw Exception(description + String(" must be string literal (in single quotes)."), ErrorCodes::BAD_ARGUMENTS);

		if (lit->value.getType() != Field::Types::String)
			throw Exception(description + String(" must be string literal (in single quotes)."), ErrorCodes::BAD_ARGUMENTS);

		return safeGet<const String &>(lit->value);
	};

	cluster_name = getClusterName(*args[0]);
	key = getStringLiteral(*args[1], "Key to hash");

	args[2] = evaluateConstantExpressionOrIdentidierAsLiteral(args[2], context);
	args[3] = evaluateConstantExpressionOrIdentidierAsLiteral(args[3], context);

	remote_database = static_cast<const ASTLiteral &>(*args[2]).value.safeGet<String>();
	remote_table = static_cast<const ASTLiteral &>(*args[3]).value.safeGet<String>();

	/// Аналогично другим TableFunctions.
	for (auto & arg : args)
		if (ASTIdentifier * id = typeid_cast<ASTIdentifier *>(arg.get()))
			id->kind = ASTIdentifier::Table;

	const Cluster & cluster = context.getCluster(cluster_name);
	size_t shard_index = sipHash64(key) % cluster.getShardCount();

	std::shared_ptr<Cluster> shard(cluster.getClusterWithSingleShard(shard_index).release());

	return StorageDistributed::create(
		getName(),
		std::make_shared<NamesAndTypesList>(getStructureOfRemoteTable(*shard, remote_database, remote_table, context)),
		remote_database,
		remote_table,
		shard,
		context);
}
예제 #2
0
파일: ZkManager.cpp 프로젝트: bradenwu/oce
	void ZkManager::notifyHandle(std::string path) {
		if( path == "" ){
			std::set<std::string> failClusterSet = notifyChangeAll();
			if( failClusterSet.size() > 0 ) {
				addFailCluster( failClusterSet );
			}
		}else { 
			std::string clusterName = getClusterName(path);
			if( notifyChange( clusterName ) == false ) {
				addFailCluster( clusterName );
			}
		}
	}
예제 #3
0
int addBinPartner(mapper_t map, char *fileName) {
	int                 res = 0;
	int                 size, i;
	char               *buff;
	struct bin_partner *partnerArr;
	int                 rangeNum;
	char               *clusterName = NULL;
	
	cluster_entry_t *cEnt;
	mapper_cluster_info_t ci;
	mapper_range_info_t   ri;
	
	    
	
	buff = getMapSourceBuff(fileName, 0, INPUT_FILE_BIN, &size);
	if(!buff) {
		debug_lr(MAP_DEBUG, "Error reading file\n");
		goto out;
	}
	if(size == 0 || (size % sizeof(struct bin_partner)) != 0) {
		debug_lr(MAP_DEBUG, "Error number of records is not integer (%d %d)\n",
			 size, sizeof(struct bin_partner));
		goto out;
	}

	clusterName = getClusterName(fileName);
	partnerArr = (struct bin_partner *) buff;
	rangeNum = size / sizeof(struct bin_partner);

	// Adding the cluster using the first range
	ci.ci_prio      =  partnerArr[0].priority;
	ci.ci_cango     =  !partnerArr[0].dontgo;
	ci.ci_cantake   =  !partnerArr[0].donttake;
	ci.ci_canexpend = 0;
	ci.ci_desc      = NULL;
	
	clusterName = getClusterName(fileName);
	
	if(!mapper_addCluster(map, -1, clusterName, &ci)) {
		debug_lr(MAP_DEBUG, "Failed to add cluster %s\n", clusterName);
		goto out;
	}
	// Checking that the cluster is not already there
	cEnt = get_cluster_entry_by_name(map, clusterName);
	if(!cEnt) {
		debug_lr(MAP_DEBUG, "Just added cluster is not there\n");
		goto out;
	}

	// Adding all the ranges
	int base = 1;
	for(i = 0 ; i < rangeNum ; i++) {
		ri.ri_core        =  0;
		ri.ri_participate =  1;
		ri.ri_proximate   =  partnerArr[i].proximate;

		struct in_addr addr;
		addr.s_addr = htonl(partnerArr[i].ip);

		//printf("Got range %s %d\n", inet_ntoa(addr), partnerArr[i].n);
		// If the range is valid adding it to the cluster (the single cluster we have)
		if(mapper_addClusterAddrRange(map, cEnt, base, &addr, partnerArr[i].n, &ri))
		{
			base += partnerArr[i].n;
		}
		else {
			goto out;
		}
	}
	
	res = 1;
 out:
	if(buff)
		free(buff);
	return res;
}
예제 #4
0
int addTextPartner(mapper_t map, char *fileName) {

	int              res = 0;
	int              size;
	int              linenum=0;
	char            *buffCurrPos = NULL;
	char            *buff = NULL;

	char            *clusterName = NULL;
	cluster_entry_t *cEnt = NULL;
	mapper_cluster_info_t ci;
	mapper_range_info_t   ri;
	int              lineBuffSize = 200;
	char             lineBuff[200];

	char             host_name[255];
	    
	cluster_info_init(&ci);
	
	buff = getMapSourceBuff(fileName, 0, INPUT_FILE, &size);
	if(!buff) {
		debug_lr(MAP_DEBUG, "Error reading file\n");
		goto out;
	}
	buffCurrPos = (char*) buff;
	if(!buffCurrPos)
		goto out;

	// Reading the cluster commet/description
	if(lineBuffSize > size)
		lineBuffSize = size;
	buffCurrPos = buff_get_line(lineBuff, lineBuffSize, buffCurrPos);
	if(!buffCurrPos) {
		debug_lr(MAP_DEBUG, "First line is not ok\n");
		goto out;
	}
	
	linenum++;
	debug_lg(MAP_DEBUG, "Processing line: %s\n", lineBuff);

	ci.ci_desc = strdup(lineBuff);
	
	
	// Reading the cluster prio, dont-take dontgo...
	buffCurrPos = buff_get_line(lineBuff, lineBuffSize, buffCurrPos);
	if(!buffCurrPos) {
		debug_lr(MAP_DEBUG, "Second line is ok\n");
		goto out;
	}
	
	linenum++;
	debug_lg(MAP_DEBUG, "Processing line: %s\n", lineBuff);

	int cango, cantake, canexpend;
	if(sscanf(lineBuff, "%d %d %d %d",
		  &ci.ci_prio, &cango, &cantake, &canexpend) != 4)
		
	{
		debug_lr(MAP_DEBUG, "Second line is not in correct format\n");
		goto out;
	}
	ci.ci_cango     = (char)cango;
	ci.ci_cantake   = (char)cantake;
	ci.ci_canexpend = (char)canexpend;
	
	clusterName = getClusterName(fileName);
	
	if(!mapper_addCluster(map, -1, clusterName, &ci)) {
		debug_lr(MAP_DEBUG, "Failed to add cluster %s\n", clusterName);
		goto out;
	}
	free(ci.ci_desc);
	ci.ci_desc = NULL;
	
	// Checking that the cluster is not already there
	cEnt = get_cluster_entry_by_name(map, clusterName);
	if(!cEnt) {
		debug_lr(MAP_DEBUG, "Just added cluster is not there\n");
		goto out;
	}

	// Reading the ranges
	int base = 1;
	while( buffCurrPos && !(*buffCurrPos == '\0') )
	{
		linenum++;
		buffCurrPos = buff_get_line(lineBuff, lineBuffSize, buffCurrPos);
		debug_lg(MAP_DEBUG, "Processing line: %s\n", lineBuff);
		

		// Skeeing comments in map
		if(lineBuff[0] == '#')
		{
			debug_lg(MAP_DEBUG, "\t\tcomment\n");
			continue;
		}
		// Skeeping empty lines
		if(strlen(lineBuff) == 0)
			continue;
		
		int count, core, participate, proximate;
		if (sscanf(lineBuff, "%s %d %d %d %d",
			   host_name, &count, &core, &participate, &proximate) == 5) {
			debug_lb(MAP_DEBUG, "Found partner range entry %s %d\n",
				 host_name, count);
                        
			ri.ri_core        = (char) core;
			ri.ri_participate = (char) participate;
			ri.ri_proximate   = (char) proximate;
			
			// If the range is valid adding it to the cluster (the
			// single cluster we have)
			// Doing the initialization part only in the first time
			// the map become old
			
			if(mapper_addClusterHostRange(map, cEnt, base, host_name, count, &ri))
			{
				base += count;
			}
			else {
				goto out;
			}
		}
		else {
			lineBuff[strlen(lineBuff)] = '\0';
			debug_lr(MAP_DEBUG, "Found bad line %s\n", lineBuff);
			sprintf(map->errorMsg, "Bad line <%s>\n", lineBuff);
			goto out;
		}
	}
	
	res = 1;
	
 out:
        if(buff)
		free(buff);
	cluster_info_free(&ci);
	return res;
}
예제 #5
0
std::string DynomiteConfig::generateConfigFile() {
    std::string localIp = getIpAddress();
    LOG( INFO ) << "Get local ip address [" << localIp << "].";

    std::string filename = getRandomTempFile();
    LOG( INFO ) << "Generate temporary configuration at [" << filename << "].";

    int token = std::rand();
    LOG( INFO ) << "Generate dynomite token [" << token << "].";

    std::string serviceName = getClusterName();
    LOG( INFO ) << "Get dynomite service name [" << serviceName << "].";

    int port = getDynomitePort();
    LOG( INFO ) << "Get dynomite port [" << port << "].";

    consul_datacenter dc;
    int ret = client->getDefaultDatacenter( dc );
    if( ret != 0 ) {
        LOG( ERROR ) << "Error in getting data center information from consul:" << ret;
        exit( ret );
    }

    std::stringstream ss;
    ss << localIp << ":" << port << ":default-rack-" << toLowerCase( dc.name )
        << ":" << dc.name << ":" << token;
    std::string tagId = ss.str();

    LOG( INFO ) << "Generate tag id [" << tagId << "].";

    consul_service dynomiteService;
    dynomiteService.service_id = serviceName + "_" + localIp;
    dynomiteService.service_name = serviceName;
    dynomiteService.service_address = localIp;
    dynomiteService.service_port = port;

    std::vector<consul_service> services;
    client->getService( serviceName, services );

    std::string seed = "";
    if( !services.empty() ) {
        std::vector<consul_service>::iterator it =
                std::find_if( services.begin(), services.end(),
                []( consul_service cs ) -> bool {
                    return cs.tags.find( "seed" ) != cs.tags.end();
                } );
        if( it != services.end() ){
            std::set<std::string>::iterator tagIt =
                    std::find_if( it->tags.begin(), it->tags.end(),
                    []( std::string tag ) -> bool {
                        return tag != "seed";
                    } );
            seed = *tagIt;
        }
    }

    std::ofstream ofstream;
    ofstream.open( filename );

    ofstream << getClusterName() << ":" << std::endl;
    ofstream << "  datacenter: " << dc.name << std::endl;
    ofstream << "  rack: default-rack-" << toLowerCase( dc.name ) << std::endl;
    ofstream << "  dyn_listen: 0.0.0.0:" << port << std::endl;
    ofstream << "  listen: 0.0.0.0:" << ( port + 1 ) << std::endl;
    ofstream << "  servers:" << std::endl;
    ofstream << "  - 127.0.0.1:6379:1" << std::endl;
    ofstream << "  tokens: '" << token << "'" << std::endl;
    ofstream << "  redis: true" << std::endl;
    ofstream << "  secure_server_option: datacenter" << std::endl;
    ofstream << "  pem_key_file: /var/dynomite/dynomite.pem" << std::endl;

    if( seed.empty() ) {
        LOG( INFO ) << "No seed server, start as seed server.";
        dynomiteService.tags.insert( "seed" );
        dynomiteService.tags.insert( tagId );
    } else {
        LOG( INFO ) << "Find seed [" << seed << "]";
        ofstream << "  dyn_seeds:" << std::endl;
        ofstream << "  - " << seed << std::endl;
        dynomiteService.tags.insert( tagId );
    }

    ofstream.close();

    ret = client->registerService( dynomiteService );
    if( ret != 0 ){
        LOG( ERROR ) << "Error to register dynomite service, return " << ret;
        exit( ret );
    }

    return filename;
}