static int storage_trunk_restore(const int64_t restore_offset) { int64_t trunk_binlog_size; int64_t line_count; TrunkBinLogReader reader; TrunkBinLogRecord record; char trunk_mark_filename[MAX_PATH_SIZE]; char buff[256]; int record_length; int result; AVLTreeInfo tree_info_by_offset; struct fast_mblock_node *pMblockNode; FDFSTrunkNode *pTrunkNode; FDFSTrunkNode trunkNode; bool trunk_init_reload_from_binlog; trunk_binlog_size = storage_trunk_get_binlog_size(); if (trunk_binlog_size < 0) { return errno != 0 ? errno : EPERM; } if (restore_offset == trunk_binlog_size) { return 0; } if (restore_offset > trunk_binlog_size) { logWarning("file: "__FILE__", line: %d, " \ "restore_offset: %"PRId64 \ " > trunk_binlog_size: %"PRId64, \ __LINE__, restore_offset, trunk_binlog_size); return storage_trunk_save(); } logDebug("file: "__FILE__", line: %d, " \ "trunk metadata recovering, start offset: " \ "%"PRId64", need recovery binlog bytes: " \ "%"PRId64, __LINE__, \ restore_offset, trunk_binlog_size - restore_offset); trunk_init_reload_from_binlog = (restore_offset == 0); if (trunk_init_reload_from_binlog) { memset(&trunkNode, 0, sizeof(trunkNode)); if ((result=avl_tree_init(&tree_info_by_offset, \ storage_trunk_free_node, \ storage_trunk_node_compare_offset)) != 0) { logError("file: "__FILE__", line: %d, " \ "avl_tree_init fail, " \ "errno: %d, error info: %s", \ __LINE__, result, STRERROR(result)); return result; } } memset(&record, 0, sizeof(record)); memset(&reader, 0, sizeof(reader)); reader.binlog_offset = restore_offset; if ((result=trunk_reader_init(NULL, &reader)) != 0) { return result; } line_count = 0; while (1) { record_length = 0; result = trunk_binlog_read(&reader, &record, &record_length); if (result != 0) { if (result == ENOENT) { if (record_length > 0) //skip incorrect record { line_count++; reader.binlog_offset += record_length; continue; } result = (reader.binlog_offset >= \ trunk_binlog_size) ? 0 : EINVAL; if (result != 0) { logError("file: "__FILE__", line: %d, " \ "binlog offset: %"PRId64 \ " < binlog size: %"PRId64 \ ", please check the end of trunk " \ "binlog", __LINE__, \ reader.binlog_offset, trunk_binlog_size); } } break; } line_count++; if (record.op_type == TRUNK_OP_TYPE_ADD_SPACE) { record.trunk.status = FDFS_TRUNK_STATUS_FREE; if (trunk_init_reload_from_binlog) { pMblockNode = fast_mblock_alloc(&free_blocks_man); if (pMblockNode == NULL) { result = errno != 0 ? errno : EIO; logError("file: "__FILE__", line: %d, "\ "malloc %d bytes fail, " \ "errno: %d, error info: %s", \ __LINE__, \ (int)sizeof(FDFSTrunkNode), \ result, STRERROR(result)); return result; } pTrunkNode = (FDFSTrunkNode *)pMblockNode->data; memcpy(&pTrunkNode->trunk, &(record.trunk), \ sizeof(FDFSTrunkFullInfo)); pTrunkNode->pMblockNode = pMblockNode; pTrunkNode->next = NULL; result = avl_tree_insert(&tree_info_by_offset,\ pTrunkNode); if (result < 0) //error { result *= -1; logError("file: "__FILE__", line: %d, "\ "avl_tree_insert fail, " \ "errno: %d, error info: %s", \ __LINE__, result, \ STRERROR(result)); return result; } else if (result == 0) { trunk_info_dump(&(record.trunk), \ buff, sizeof(buff)); logWarning("file: "__FILE__", line: %d"\ ", trunk data line: " \ "%"PRId64", trunk "\ "space already exist, "\ "trunk info: %s", \ __LINE__, line_count, buff); } } else if ((result=trunk_add_space_by_trunk( \ &record.trunk)) != 0) { break; } } else if (record.op_type == TRUNK_OP_TYPE_DEL_SPACE) { record.trunk.status = FDFS_TRUNK_STATUS_FREE; if (trunk_init_reload_from_binlog) { memcpy(&trunkNode.trunk, &record.trunk, \ sizeof(FDFSTrunkFullInfo)); if (avl_tree_delete(&tree_info_by_offset,\ &trunkNode) != 1) { trunk_info_dump(&(record.trunk), \ buff, sizeof(buff)); logWarning("file: "__FILE__", line: %d"\ ", binlog offset: %"PRId64 \ ", trunk data line: %"PRId64 \ " trunk node not exist, " \ "trunk info: %s", __LINE__, \ reader.binlog_offset, \ line_count, buff); } } else if ((result=trunk_delete_space( \ &record.trunk, false)) != 0) { if (result == ENOENT) { logDebug("file: "__FILE__", line: %d, "\ "binlog offset: %"PRId64 \ ", trunk data line: %"PRId64,\ __LINE__, reader.binlog_offset, \ line_count); result = 0; } else { break; } } } reader.binlog_offset += record_length; } trunk_reader_destroy(&reader); trunk_mark_filename_by_reader(&reader, trunk_mark_filename); if (unlink(trunk_mark_filename) != 0) { logError("file: "__FILE__", line: %d, " \ "unlink file %s fail, " \ "errno: %d, error info: %s", __LINE__, \ trunk_mark_filename, errno, STRERROR(errno)); } if (result != 0) { if (trunk_init_reload_from_binlog) { avl_tree_destroy(&tree_info_by_offset); } logError("file: "__FILE__", line: %d, " \ "trunk load fail, errno: %d, error info: %s", \ __LINE__, result, STRERROR(result)); return result; } if (trunk_init_reload_from_binlog) { logInfo("file: "__FILE__", line: %d, " \ "free tree node count: %d", \ __LINE__, avl_tree_count(&tree_info_by_offset)); result = avl_tree_walk(&tree_info_by_offset, \ storage_trunk_add_free_blocks_callback, NULL); tree_info_by_offset.free_data_func = NULL; avl_tree_destroy(&tree_info_by_offset); } if (result == 0) { logDebug("file: "__FILE__", line: %d, " \ "trunk metadata recovery done. start offset: " \ "%"PRId64", recovery file size: " \ "%"PRId64, __LINE__, \ restore_offset, trunk_binlog_size - restore_offset); return storage_trunk_save(); } return result; }
static void* trunk_sync_thread_entrance(void* arg) { FDFSStorageBrief *pStorage; TrunkBinLogReader reader; ConnectionInfo storage_server; char local_ip_addr[IP_ADDRESS_SIZE]; int read_result; int sync_result; int conn_result; int result; int previousCode; int nContinuousFail; time_t current_time; time_t last_keep_alive_time; memset(local_ip_addr, 0, sizeof(local_ip_addr)); memset(&reader, 0, sizeof(reader)); reader.mark_fd = -1; reader.binlog_fd = -1; current_time = g_current_time; last_keep_alive_time = 0; pStorage = (FDFSStorageBrief *)arg; strcpy(storage_server.ip_addr, pStorage->ip_addr); storage_server.port = g_server_port; storage_server.sock = -1; logInfo("file: "__FILE__", line: %d, " \ "trunk sync thread to storage server %s:%d started", \ __LINE__, storage_server.ip_addr, storage_server.port); while (g_continue_flag && g_if_trunker_self && \ pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ pStorage->status != FDFS_STORAGE_STATUS_NONE) { previousCode = 0; nContinuousFail = 0; conn_result = 0; while (g_continue_flag && g_if_trunker_self && \ pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ pStorage->status != FDFS_STORAGE_STATUS_NONE) { strcpy(storage_server.ip_addr, pStorage->ip_addr); storage_server.sock = \ socket(AF_INET, SOCK_STREAM, 0); if(storage_server.sock < 0) { logCrit("file: "__FILE__", line: %d," \ " socket create fail, " \ "errno: %d, error info: %s. " \ "program exit!", __LINE__, \ errno, STRERROR(errno)); g_continue_flag = false; break; } if (g_client_bind_addr && *g_bind_addr != '\0') { socketBind(storage_server.sock, g_bind_addr, 0); } if (tcpsetnonblockopt(storage_server.sock) != 0) { nContinuousFail++; close(storage_server.sock); storage_server.sock = -1; sleep(1); continue; } if ((conn_result=connectserverbyip_nb(storage_server.sock,\ pStorage->ip_addr, g_server_port, \ g_fdfs_connect_timeout)) == 0) { char szFailPrompt[64]; if (nContinuousFail == 0) { *szFailPrompt = '\0'; } else { sprintf(szFailPrompt, \ ", continuous fail count: %d", \ nContinuousFail); } logInfo("file: "__FILE__", line: %d, " \ "successfully connect to " \ "storage server %s:%d%s", __LINE__, \ pStorage->ip_addr, g_server_port, \ szFailPrompt); nContinuousFail = 0; break; } if (previousCode != conn_result) { logError("file: "__FILE__", line: %d, " \ "connect to storage server %s:%d fail" \ ", errno: %d, error info: %s", \ __LINE__, \ pStorage->ip_addr, g_server_port, \ conn_result, STRERROR(conn_result)); previousCode = conn_result; } nContinuousFail++; close(storage_server.sock); storage_server.sock = -1; if (!g_continue_flag) { break; } sleep(1); } if (nContinuousFail > 0) { logError("file: "__FILE__", line: %d, " \ "connect to storage server %s:%d fail, " \ "try count: %d, errno: %d, error info: %s", \ __LINE__, pStorage->ip_addr, \ g_server_port, nContinuousFail, \ conn_result, STRERROR(conn_result)); } if ((!g_continue_flag) || (!g_if_trunker_self) || \ pStorage->status == FDFS_STORAGE_STATUS_DELETED || \ pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ pStorage->status == FDFS_STORAGE_STATUS_NONE) { logError("file: "__FILE__", line: %d, break loop." \ "g_continue_flag: %d, g_if_trunker_self: %d, " \ "dest storage status: %d", __LINE__, \ g_continue_flag, g_if_trunker_self, \ pStorage->status); break; } if ((result=trunk_reader_init(pStorage, &reader)) != 0) { logCrit("file: "__FILE__", line: %d, " \ "trunk_reader_init fail, errno=%d, " \ "program exit!", \ __LINE__, result); g_continue_flag = false; break; } getSockIpaddr(storage_server.sock, \ local_ip_addr, IP_ADDRESS_SIZE); insert_into_local_host_ip(local_ip_addr); /* //printf("file: "__FILE__", line: %d, " \ "storage_server.ip_addr=%s, " \ "local_ip_addr: %s\n", \ __LINE__, pStorage->ip_addr, local_ip_addr); */ if (is_local_host_ip(pStorage->ip_addr)) { //can't self sync to self logError("file: "__FILE__", line: %d, " \ "ip_addr %s belong to the local host," \ " trunk sync thread exit.", \ __LINE__, pStorage->ip_addr); fdfs_quit(&storage_server); close(storage_server.sock); break; } if (reader.binlog_offset == 0) { if ((result=fdfs_deal_no_body_cmd(&storage_server, \ STORAGE_PROTO_CMD_TRUNK_TRUNCATE_BINLOG_FILE)) != 0) { logError("file: "__FILE__", line: %d, " "fdfs_deal_no_body_cmd fail, result: %d", __LINE__, result); close(storage_server.sock); trunk_reader_destroy(&reader); sleep(5); continue; } } sync_result = 0; while (g_continue_flag && \ pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ pStorage->status != FDFS_STORAGE_STATUS_NONE) { read_result = trunk_binlog_preread(&reader); if (read_result == ENOENT) { if (reader.last_binlog_offset != \ reader.binlog_offset) { if (trunk_write_to_mark_file(&reader)!=0) { logCrit("file: "__FILE__", line: %d, " \ "trunk_write_to_mark_file fail, " \ "program exit!", __LINE__); g_continue_flag = false; break; } } current_time = g_current_time; if (current_time - last_keep_alive_time >= \ g_heart_beat_interval) { if (fdfs_active_test(&storage_server)!=0) { break; } last_keep_alive_time = current_time; } if (!g_if_trunker_self) { break; } usleep(g_sync_wait_usec); continue; } if (read_result != 0) { sleep(5); continue; } if ((sync_result=trunk_sync_data(&reader, \ &storage_server)) != 0) { break; } if (g_sync_interval > 0) { usleep(g_sync_interval); } } if (reader.last_binlog_offset != reader.binlog_offset) { if (trunk_write_to_mark_file(&reader) != 0) { logCrit("file: "__FILE__", line: %d, " \ "trunk_write_to_mark_file fail, " \ "program exit!", __LINE__); g_continue_flag = false; break; } } close(storage_server.sock); storage_server.sock = -1; trunk_reader_destroy(&reader); if (!g_continue_flag) { break; } if (!(sync_result == ENOTCONN || sync_result == EIO)) { sleep(1); } } if (storage_server.sock >= 0) { close(storage_server.sock); } trunk_reader_destroy(&reader); trunk_sync_thread_exit(&storage_server); return NULL; }