static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo) { char buff[32]; int result; int filename_len; char short_filename[64]; char full_filename[MAX_PATH_SIZE]; int sub_path_high; int sub_path_low; while (1) { pthread_mutex_lock(&trunk_file_lock); pTrunkInfo->file.id = ++g_current_trunk_file_id; result = storage_write_to_sync_ini_file(); pthread_mutex_unlock(&trunk_file_lock); if (result != 0) { return result; } int2buff(pTrunkInfo->file.id, buff); base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(int), \ short_filename, &filename_len, false); storage_get_store_path(short_filename, filename_len, \ &sub_path_high, &sub_path_low); pTrunkInfo->path.sub_path_high = sub_path_high; pTrunkInfo->path.sub_path_low = sub_path_low; trunk_get_full_filename(pTrunkInfo, full_filename, \ sizeof(full_filename)); if (!fileExists(full_filename)) { break; } } if ((result=trunk_init_file(full_filename)) != 0) { return result; } return 0; }
static bool storage_trunk_is_space_occupied(const FDFSTrunkFullInfo *pTrunkInfo) { int result; int fd; char full_filename[MAX_PATH_SIZE+64]; trunk_get_full_filename(pTrunkInfo, full_filename, sizeof(full_filename)); fd = open(full_filename, O_RDONLY, 0644); if (fd < 0) { result = errno != 0 ? errno : ENOENT; logWarning("file: " __FILE__ ", line: %d, " "open file: %s fail, " \ "errno: %d, error info: %s", \ __LINE__, full_filename, \ result, STRERROR(result)); return false; } if (pTrunkInfo->file.offset > 0 && lseek(fd, pTrunkInfo->file.offset, \ SEEK_SET) < 0) { result = errno != 0 ? errno : EIO; logError("file: "__FILE__", line: %d, " \ "lseek file: %s fail, " \ "errno: %d, error info: %s", \ __LINE__, full_filename, \ result, STRERROR(result)); close(fd); return false; } /* logInfo("fd: %d, trunk filename: %s, offset: %d", fd, full_filename, \ pTrunkInfo->file.offset); */ result = dio_check_trunk_file_ex(fd, full_filename, \ pTrunkInfo->file.offset); close(fd); return (result == EEXIST); }
static int storage_do_recovery(const char *pBasePath, StorageBinLogReader *pReader, \ ConnectionInfo *pSrcStorage) { ConnectionInfo *pTrackerServer; ConnectionInfo *pStorageConn; FDFSTrunkFullInfo trunk_info; StorageBinLogRecord record; int record_length; int result; int log_level; int count; int store_path_index; int64_t file_size; int64_t total_count; int64_t success_count; bool bContinueFlag; char local_filename[MAX_PATH_SIZE]; char src_filename[MAX_PATH_SIZE]; pTrackerServer = g_tracker_group.servers; count = 0; total_count = 0; success_count = 0; result = 0; logInfo("file: "__FILE__", line: %d, " \ "disk recovery: recovering files of data path: %s ...", \ __LINE__, pBasePath); bContinueFlag = true; while (bContinueFlag) { if ((pStorageConn=tracker_connect_server(pSrcStorage, &result)) == NULL) { sleep(5); continue; } while (g_continue_flag) { result=storage_binlog_read(pReader, &record, &record_length); if (result != 0) { if (result == ENOENT) { result = 0; } bContinueFlag = false; break; } total_count++; if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_FILE) { bool bTrunkFile; if (fdfs_is_trunk_file(record.filename, \ record.filename_len)) { char *pTrunkPathEnd; char *pLocalFilename; bTrunkFile = true; if (fdfs_decode_trunk_info(record.store_path_index, \ record.true_filename, record.true_filename_len,\ &trunk_info) != 0) { pReader->binlog_offset += record_length; count++; continue; } trunk_get_full_filename(&trunk_info, \ local_filename, sizeof(local_filename)); pTrunkPathEnd = strrchr(record.filename, '/'); pLocalFilename = strrchr(local_filename, '/'); if (pTrunkPathEnd == NULL || pLocalFilename == NULL) { pReader->binlog_offset += record_length; count++; continue; } sprintf(pTrunkPathEnd + 1, "%s", pLocalFilename + 1); } else { bTrunkFile = false; sprintf(local_filename, "%s/data/%s", \ g_fdfs_store_paths.paths[record.store_path_index], \ record.true_filename); } result = storage_download_file_to_file(pTrackerServer, \ pStorageConn, g_group_name, \ record.filename, local_filename, \ &file_size); if (result == 0) { if (!bTrunkFile) { set_file_utimes(local_filename, \ record.timestamp); } success_count++; } else if (result != ENOENT) { break; } } else if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_LINK) { if (record.src_filename_len == 0) { logError("file: "__FILE__", line: %d, " \ "invalid binlog line, filename: %s, " \ "expect src filename", __LINE__, \ record.filename); result = EINVAL; bContinueFlag = false; break; } if ((result=storage_split_filename_ex(record.filename, \ &record.filename_len, record.true_filename, \ &store_path_index)) != 0) { bContinueFlag = false; break; } sprintf(local_filename, "%s/data/%s", \ g_fdfs_store_paths.paths[store_path_index], \ record.true_filename); if ((result=storage_split_filename_ex( \ record.src_filename, &record.src_filename_len,\ record.true_filename, &store_path_index)) != 0) { bContinueFlag = false; break; } sprintf(src_filename, "%s/data/%s", \ g_fdfs_store_paths.paths[store_path_index], \ record.true_filename); if (symlink(src_filename, local_filename) == 0) { success_count++; } else { result = errno != 0 ? errno : ENOENT; if (result == ENOENT || result == EEXIST) { log_level = LOG_DEBUG; } else { log_level = LOG_ERR; } log_it_ex(&g_log_context, log_level, \ "file: "__FILE__", line: %d, " \ "link file %s to %s fail, " \ "errno: %d, error info: %s", __LINE__,\ src_filename, local_filename, \ result, STRERROR(result)); if (result != ENOENT && result != EEXIST) { bContinueFlag = false; break; } } } else { logError("file: "__FILE__", line: %d, " \ "invalid file op type: %d", \ __LINE__, record.op_type); result = EINVAL; bContinueFlag = false; break; } pReader->binlog_offset += record_length; count++; if (count == 1000) { logDebug("file: "__FILE__", line: %d, " \ "disk recovery: recover path: %s, " \ "file count: %"PRId64 \ ", success count: %"PRId64, \ __LINE__, pBasePath, total_count, \ success_count); recovery_write_to_mark_file(pBasePath, pReader); count = 0; } } tracker_disconnect_server_ex(pStorageConn, result != 0); if (count > 0) { recovery_write_to_mark_file(pBasePath, pReader); count = 0; logInfo("file: "__FILE__", line: %d, " \ "disk recovery: recover path: %s, " \ "file count: %"PRId64 \ ", success count: %"PRId64, \ __LINE__, pBasePath, total_count, success_count); } else { sleep(5); } } if (result == 0) { logInfo("file: "__FILE__", line: %d, " \ "disk recovery: recover files of data path: %s done", \ __LINE__, pBasePath); } return result; }
static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo) { char buff[32]; int i; int result; int filename_len; char short_filename[64]; char full_filename[MAX_PATH_SIZE]; int store_path_index; int sub_path_high; int sub_path_low; store_path_index = g_store_path_index; if (g_store_path_mode == FDFS_STORE_PATH_LOAD_BALANCE) { if (store_path_index < 0) { return ENOSPC; } } else { if (store_path_index >= g_fdfs_store_paths.count) { store_path_index = 0; } if (!storage_check_reserved_space_path(g_path_space_list \ [store_path_index].total_mb, g_path_space_list \ [store_path_index].free_mb, g_avg_storage_reserved_mb)) { for (i=0; i<g_fdfs_store_paths.count; i++) { if (storage_check_reserved_space_path( \ g_path_space_list[i].total_mb, \ g_path_space_list[i].free_mb, \ g_avg_storage_reserved_mb)) { store_path_index = i; g_store_path_index = i; break; } } if (i == g_fdfs_store_paths.count) { return ENOSPC; } } g_store_path_index++; if (g_store_path_index >= g_fdfs_store_paths.count) { g_store_path_index = 0; } } pTrunkInfo->path.store_path_index = store_path_index; while (1) { pthread_mutex_lock(&trunk_file_lock); pTrunkInfo->file.id = ++g_current_trunk_file_id; result = storage_write_to_sync_ini_file(); pthread_mutex_unlock(&trunk_file_lock); if (result != 0) { return result; } int2buff(pTrunkInfo->file.id, buff); base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(int), \ short_filename, &filename_len, false); storage_get_store_path(short_filename, filename_len, \ &sub_path_high, &sub_path_low); pTrunkInfo->path.sub_path_high = sub_path_high; pTrunkInfo->path.sub_path_low = sub_path_low; trunk_get_full_filename(pTrunkInfo, full_filename, \ sizeof(full_filename)); if (!fileExists(full_filename)) { break; } } if ((result=trunk_init_file(full_filename)) != 0) { return result; } return 0; }