/*插入到g_local_host_ip_addrs中*/
int insert_into_local_host_ip(const char *client_ip)
{
/*去重*/
if (is_local_host_ip(client_ip))
{
return 0;
}
if (g_local_host_ip_count >= STORAGE_MAX_LOCAL_IP_ADDRS)
{
return -1;
}
/*存放之*/
strcpy(g_local_host_ip_addrs + \
IP_ADDRESS_SIZE * g_local_host_ip_count, \
client_ip);
g_local_host_ip_count++;
return 1;
}
static int relationship_select_leader()
{
int result;
TrackerRunningStatus trackerStatus;
if (g_tracker_servers.server_count <= 0)
{
return 0;
}
logInfo("file: "__FILE__", line: %d, " \
"selecting leader...", __LINE__);
if ((result=relationship_get_tracker_leader(&trackerStatus)) != 0)
{
return result;
}
if (trackerStatus.pTrackerServer->port == g_server_port && \
is_local_host_ip(trackerStatus.pTrackerServer->ip_addr))
{
if ((result=relationship_notify_leader_changed( \
trackerStatus.pTrackerServer)) != 0)
{
return result;
}
logInfo("file: "__FILE__", line: %d, " \
"I am the new tracker leader %s:%d", \
__LINE__, trackerStatus.pTrackerServer->ip_addr, \
trackerStatus.pTrackerServer->port);
tracker_mem_find_trunk_servers();
}
else
{
if (trackerStatus.if_leader)
{
g_tracker_servers.leader_index = \
trackerStatus.pTrackerServer - \
g_tracker_servers.servers;
if (g_tracker_servers.leader_index < 0 || \
g_tracker_servers.leader_index >= \
g_tracker_servers.server_count)
{
g_tracker_servers.leader_index = -1;
return EINVAL;
}
logInfo("file: "__FILE__", line: %d, " \
"the tracker leader %s:%d", __LINE__, \
trackerStatus.pTrackerServer->ip_addr, \
trackerStatus.pTrackerServer->port);
}
else
{
logDebug("file: "__FILE__", line: %d, " \
"waiting for leader notify", __LINE__);
return ENOENT;
}
}
return 0;
}
int trunk_sync_thread_start(const FDFSStorageBrief *pStorage)
{
int result;
pthread_attr_t pattr;
pthread_t tid;
if (pStorage->status == FDFS_STORAGE_STATUS_DELETED || \
pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \
pStorage->status == FDFS_STORAGE_STATUS_NONE)
{
return 0;
}
if (is_local_host_ip(pStorage->ip_addr)) //can't self sync to self
{
return 0;
}
if ((result=init_pthread_attr(&pattr, g_thread_stack_size)) != 0)
{
return result;
}
/*
//printf("start storage ip_addr: %s, g_trunk_sync_thread_count=%d\n",
pStorage->ip_addr, g_trunk_sync_thread_count);
*/
if ((result=pthread_create(&tid, &pattr, trunk_sync_thread_entrance, \
(void *)pStorage)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"create thread failed, errno: %d, " \
"error info: %s", \
__LINE__, result, STRERROR(result));
pthread_attr_destroy(&pattr);
return result;
}
if ((result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"call pthread_mutex_lock fail, " \
"errno: %d, error info: %s", \
__LINE__, result, STRERROR(result));
}
g_trunk_sync_thread_count++;
trunk_sync_tids = (pthread_t *)realloc(trunk_sync_tids, sizeof(pthread_t) * \
g_trunk_sync_thread_count);
if (trunk_sync_tids == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", \
__LINE__, (int)sizeof(pthread_t) * \
g_trunk_sync_thread_count, \
errno, STRERROR(errno));
}
else
{
trunk_sync_tids[g_trunk_sync_thread_count - 1] = tid;
}
if ((result=pthread_mutex_unlock(&trunk_sync_thread_lock)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"call pthread_mutex_unlock fail, " \
"errno: %d, error info: %s", \
__LINE__, result, STRERROR(result));
}
pthread_attr_destroy(&pattr);
return 0;
}
static int do_notify_leader_changed(ConnectionInfo *pTrackerServer, \
ConnectionInfo *pLeader, const char cmd, bool *bConnectFail)
{
char out_buff[sizeof(TrackerHeader) + FDFS_PROTO_IP_PORT_SIZE];
char in_buff[1];
ConnectionInfo *conn;
TrackerHeader *pHeader;
char *pInBuff;
int64_t in_bytes;
int result;
pTrackerServer->sock = -1;
if ((conn=tracker_connect_server(pTrackerServer, &result)) == NULL)
{
*bConnectFail = true;
return result;
}
*bConnectFail = false;
do
{
memset(out_buff, 0, sizeof(out_buff));
pHeader = (TrackerHeader *)out_buff;
pHeader->cmd = cmd;
sprintf(out_buff + sizeof(TrackerHeader), "%s:%d", \
pLeader->ip_addr, pLeader->port);
long2buff(FDFS_PROTO_IP_PORT_SIZE, pHeader->pkg_len);
if ((result=tcpsenddata_nb(conn->sock, out_buff, \
sizeof(out_buff), g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"send data to tracker server %s:%d fail, " \
"errno: %d, error info: %s", __LINE__, \
pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
result = (result == ENOENT ? EACCES : result);
break;
}
pInBuff = in_buff;
result = fdfs_recv_response(conn, &pInBuff, \
0, &in_bytes);
if (result != 0)
{
break;
}
if (in_bytes != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d response data " \
"length: %"PRId64" is invalid, " \
"expect length: %d.", __LINE__, \
pTrackerServer->ip_addr, pTrackerServer->port, \
in_bytes, 0);
result = EINVAL;
break;
}
} while (0);
if (pTrackerServer->port == g_server_port && \
is_local_host_ip(pTrackerServer->ip_addr))
{
tracker_disconnect_server_ex(conn, true);
}
else
{
tracker_disconnect_server_ex(conn, result != 0);
}
return result;
}
static void* trunk_sync_thread_entrance(void* arg)
{
FDFSStorageBrief *pStorage;
TrunkBinLogReader reader;
ConnectionInfo storage_server;
char local_ip_addr[IP_ADDRESS_SIZE];
int read_result;
int sync_result;
int conn_result;
int result;
int previousCode;
int nContinuousFail;
time_t current_time;
time_t last_keep_alive_time;
memset(local_ip_addr, 0, sizeof(local_ip_addr));
memset(&reader, 0, sizeof(reader));
reader.mark_fd = -1;
reader.binlog_fd = -1;
current_time = g_current_time;
last_keep_alive_time = 0;
pStorage = (FDFSStorageBrief *)arg;
strcpy(storage_server.ip_addr, pStorage->ip_addr);
storage_server.port = g_server_port;
storage_server.sock = -1;
logInfo("file: "__FILE__", line: %d, " \
"trunk sync thread to storage server %s:%d started", \
__LINE__, storage_server.ip_addr, storage_server.port);
while (g_continue_flag && g_if_trunker_self && \
pStorage->status != FDFS_STORAGE_STATUS_DELETED && \
pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \
pStorage->status != FDFS_STORAGE_STATUS_NONE)
{
previousCode = 0;
nContinuousFail = 0;
conn_result = 0;
while (g_continue_flag && g_if_trunker_self && \
pStorage->status != FDFS_STORAGE_STATUS_DELETED && \
pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \
pStorage->status != FDFS_STORAGE_STATUS_NONE)
{
strcpy(storage_server.ip_addr, pStorage->ip_addr);
storage_server.sock = \
socket(AF_INET, SOCK_STREAM, 0);
if(storage_server.sock < 0)
{
logCrit("file: "__FILE__", line: %d," \
" socket create fail, " \
"errno: %d, error info: %s. " \
"program exit!", __LINE__, \
errno, STRERROR(errno));
g_continue_flag = false;
break;
}
if (g_client_bind_addr && *g_bind_addr != '\0')
{
socketBind(storage_server.sock, g_bind_addr, 0);
}
if (tcpsetnonblockopt(storage_server.sock) != 0)
{
nContinuousFail++;
close(storage_server.sock);
storage_server.sock = -1;
sleep(1);
continue;
}
if ((conn_result=connectserverbyip_nb(storage_server.sock,\
pStorage->ip_addr, g_server_port, \
g_fdfs_connect_timeout)) == 0)
{
char szFailPrompt[64];
if (nContinuousFail == 0)
{
*szFailPrompt = '\0';
}
else
{
sprintf(szFailPrompt, \
", continuous fail count: %d", \
nContinuousFail);
}
logInfo("file: "__FILE__", line: %d, " \
"successfully connect to " \
"storage server %s:%d%s", __LINE__, \
pStorage->ip_addr, g_server_port, \
szFailPrompt);
nContinuousFail = 0;
break;
}
if (previousCode != conn_result)
{
logError("file: "__FILE__", line: %d, " \
"connect to storage server %s:%d fail" \
", errno: %d, error info: %s", \
__LINE__, \
pStorage->ip_addr, g_server_port, \
conn_result, STRERROR(conn_result));
previousCode = conn_result;
}
nContinuousFail++;
close(storage_server.sock);
storage_server.sock = -1;
if (!g_continue_flag)
{
break;
}
sleep(1);
}
if (nContinuousFail > 0)
{
logError("file: "__FILE__", line: %d, " \
"connect to storage server %s:%d fail, " \
"try count: %d, errno: %d, error info: %s", \
__LINE__, pStorage->ip_addr, \
g_server_port, nContinuousFail, \
conn_result, STRERROR(conn_result));
}
if ((!g_continue_flag) || (!g_if_trunker_self) || \
pStorage->status == FDFS_STORAGE_STATUS_DELETED || \
pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \
pStorage->status == FDFS_STORAGE_STATUS_NONE)
{
logError("file: "__FILE__", line: %d, break loop." \
"g_continue_flag: %d, g_if_trunker_self: %d, " \
"dest storage status: %d", __LINE__, \
g_continue_flag, g_if_trunker_self, \
pStorage->status);
break;
}
if ((result=trunk_reader_init(pStorage, &reader)) != 0)
{
logCrit("file: "__FILE__", line: %d, " \
"trunk_reader_init fail, errno=%d, " \
"program exit!", \
__LINE__, result);
g_continue_flag = false;
break;
}
getSockIpaddr(storage_server.sock, \
local_ip_addr, IP_ADDRESS_SIZE);
insert_into_local_host_ip(local_ip_addr);
/*
//printf("file: "__FILE__", line: %d, " \
"storage_server.ip_addr=%s, " \
"local_ip_addr: %s\n", \
__LINE__, pStorage->ip_addr, local_ip_addr);
*/
if (is_local_host_ip(pStorage->ip_addr))
{ //can't self sync to self
logError("file: "__FILE__", line: %d, " \
"ip_addr %s belong to the local host," \
" trunk sync thread exit.", \
__LINE__, pStorage->ip_addr);
fdfs_quit(&storage_server);
close(storage_server.sock);
break;
}
if (reader.binlog_offset == 0)
{
if ((result=fdfs_deal_no_body_cmd(&storage_server, \
STORAGE_PROTO_CMD_TRUNK_TRUNCATE_BINLOG_FILE)) != 0)
{
logError("file: "__FILE__", line: %d, "
"fdfs_deal_no_body_cmd fail, result: %d",
__LINE__, result);
close(storage_server.sock);
trunk_reader_destroy(&reader);
sleep(5);
continue;
}
}
sync_result = 0;
while (g_continue_flag && \
pStorage->status != FDFS_STORAGE_STATUS_DELETED && \
pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \
pStorage->status != FDFS_STORAGE_STATUS_NONE)
{
read_result = trunk_binlog_preread(&reader);
if (read_result == ENOENT)
{
if (reader.last_binlog_offset != \
reader.binlog_offset)
{
if (trunk_write_to_mark_file(&reader)!=0)
{
logCrit("file: "__FILE__", line: %d, " \
"trunk_write_to_mark_file fail, " \
"program exit!", __LINE__);
g_continue_flag = false;
break;
}
}
current_time = g_current_time;
if (current_time - last_keep_alive_time >= \
g_heart_beat_interval)
{
if (fdfs_active_test(&storage_server)!=0)
{
break;
}
last_keep_alive_time = current_time;
}
if (!g_if_trunker_self)
{
break;
}
usleep(g_sync_wait_usec);
continue;
}
if (read_result != 0)
{
sleep(5);
continue;
}
if ((sync_result=trunk_sync_data(&reader, \
&storage_server)) != 0)
{
break;
}
if (g_sync_interval > 0)
{
usleep(g_sync_interval);
}
}
if (reader.last_binlog_offset != reader.binlog_offset)
{
if (trunk_write_to_mark_file(&reader) != 0)
{
logCrit("file: "__FILE__", line: %d, " \
"trunk_write_to_mark_file fail, " \
"program exit!", __LINE__);
g_continue_flag = false;
break;
}
}
close(storage_server.sock);
storage_server.sock = -1;
trunk_reader_destroy(&reader);
if (!g_continue_flag)
{
break;
}
if (!(sync_result == ENOTCONN || sync_result == EIO))
{
sleep(1);
}
}
if (storage_server.sock >= 0)
{
close(storage_server.sock);
}
trunk_reader_destroy(&reader);
trunk_sync_thread_exit(&storage_server);
return NULL;
}