static int storage_trunk_save()
{
int result;
if (!(g_trunk_compress_binlog_min_interval > 0 && \
g_current_time - g_trunk_last_compress_time >
g_trunk_compress_binlog_min_interval))
{
return storage_trunk_do_save();
}
logInfo("start compress trunk binlog ...");
if ((result=trunk_binlog_compress_apply()) != 0)
{
return result;
}
if ((result=storage_trunk_do_save()) != 0)
{
trunk_binlog_compress_rollback();
return result;
}
if ((result=trunk_binlog_compress_commit()) != 0)
{
trunk_binlog_compress_rollback();
return result;
}
g_trunk_last_compress_time = g_current_time;
storage_write_to_sync_ini_file();
logInfo("compress trunk binlog done.");
return trunk_unlink_all_mark_files(); //because the binlog file be compressed
}
static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo)
{
char buff[32];
int result;
int filename_len;
char short_filename[64];
char full_filename[MAX_PATH_SIZE];
int sub_path_high;
int sub_path_low;
while (1)
{
pthread_mutex_lock(&trunk_file_lock);
pTrunkInfo->file.id = ++g_current_trunk_file_id;
result = storage_write_to_sync_ini_file();
pthread_mutex_unlock(&trunk_file_lock);
if (result != 0)
{
return result;
}
int2buff(pTrunkInfo->file.id, buff);
base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(int), \
short_filename, &filename_len, false);
storage_get_store_path(short_filename, filename_len, \
&sub_path_high, &sub_path_low);
pTrunkInfo->path.sub_path_high = sub_path_high;
pTrunkInfo->path.sub_path_low = sub_path_low;
trunk_get_full_filename(pTrunkInfo, full_filename, \
sizeof(full_filename));
if (!fileExists(full_filename))
{
break;
}
}
if ((result=trunk_init_file(full_filename)) != 0)
{
return result;
}
return 0;
}
int storage_get_params_from_tracker()
{
IniContext iniContext;
int result;
bool use_trunk_file;
char reserved_space_str[32];
char *pIdType;
if ((result=fdfs_get_ini_context_from_tracker(&g_tracker_group, \
&iniContext, (bool * volatile)&g_continue_flag, \
g_client_bind_addr, g_bind_addr)) != 0)
{
return result;
}
g_storage_ip_changed_auto_adjust = iniGetBoolValue(NULL, \
"storage_ip_changed_auto_adjust", \
&iniContext, false);
g_store_path_mode = iniGetIntValue(NULL, "store_path", &iniContext, \
FDFS_STORE_PATH_ROUND_ROBIN);
if ((result=fdfs_parse_storage_reserved_space(&iniContext, \
&g_storage_reserved_space)) != 0)
{
iniFreeContext(&iniContext);
return result;
}
if (g_storage_reserved_space.flag == \
TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB)
{
g_avg_storage_reserved_mb = g_storage_reserved_space.rs.mb \
/ g_fdfs_store_paths.count;
}
else
{
g_avg_storage_reserved_mb = 0;
}
g_use_storage_id = iniGetBoolValue(NULL, "use_storage_id", \
&iniContext, false);
use_trunk_file = iniGetBoolValue(NULL, "use_trunk_file", \
&iniContext, false);
g_slot_min_size = iniGetIntValue(NULL, "slot_min_size", \
&iniContext, 256);
g_trunk_file_size = iniGetIntValue(NULL, "trunk_file_size", \
&iniContext, 64 * 1024 * 1024);
g_slot_max_size = iniGetIntValue(NULL, "slot_max_size", \
&iniContext, g_trunk_file_size / 2);
g_trunk_create_file_advance = iniGetBoolValue(NULL, \
"trunk_create_file_advance", &iniContext, false);
if ((result=get_time_item_from_conf(&iniContext, \
"trunk_create_file_time_base", \
&g_trunk_create_file_time_base, 2, 0)) != 0)
{
iniFreeContext(&iniContext);
return result;
}
g_trunk_create_file_interval = iniGetIntValue(NULL, \
"trunk_create_file_interval", &iniContext, \
86400);
g_trunk_create_file_space_threshold = iniGetInt64Value(NULL, \
"trunk_create_file_space_threshold", \
&iniContext, 0);
g_trunk_init_check_occupying = iniGetBoolValue(NULL, \
"trunk_init_check_occupying", &iniContext, false);
g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL, \
"trunk_init_reload_from_binlog", &iniContext, false);
g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL, \
"trunk_compress_binlog_min_interval", &iniContext, 0);
g_store_slave_file_use_link = iniGetBoolValue(NULL, \
"store_slave_file_use_link", &iniContext, false);
pIdType = iniGetStrValue(NULL, "id_type_in_filename", &iniContext);
if (pIdType != NULL && strcasecmp(pIdType, "id") == 0)
{
if (g_use_storage_id)
{
g_id_type_in_filename = FDFS_ID_TYPE_SERVER_ID;
}
else
{
g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS;
}
}
else
{
g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS;
}
iniFreeContext(&iniContext);
if (use_trunk_file && !g_if_use_trunk_file)
{
if ((result=trunk_sync_init()) != 0)
{
return result;
}
}
g_if_use_trunk_file = use_trunk_file;
logInfo("file: "__FILE__", line: %d, " \
"use_storage_id=%d, " \
"id_type_in_filename=%s, " \
"storage_ip_changed_auto_adjust=%d, " \
"store_path=%d, " \
"reserved_storage_space=%s, " \
"use_trunk_file=%d, " \
"slot_min_size=%d, " \
"slot_max_size=%d MB, " \
"trunk_file_size=%d MB, " \
"trunk_create_file_advance=%d, " \
"trunk_create_file_time_base=%02d:%02d, " \
"trunk_create_file_interval=%d, " \
"trunk_create_file_space_threshold=%d GB, " \
"trunk_init_check_occupying=%d, " \
"trunk_init_reload_from_binlog=%d, " \
"trunk_compress_binlog_min_interval=%d, " \
"store_slave_file_use_link=%d", \
__LINE__, g_use_storage_id, \
g_id_type_in_filename == FDFS_ID_TYPE_SERVER_ID ? "id" : "ip", \
g_storage_ip_changed_auto_adjust, \
g_store_path_mode, fdfs_storage_reserved_space_to_string( \
&g_storage_reserved_space, reserved_space_str), \
g_if_use_trunk_file, g_slot_min_size, \
g_slot_max_size / FDFS_ONE_MB, \
g_trunk_file_size / FDFS_ONE_MB, \
g_trunk_create_file_advance, \
g_trunk_create_file_time_base.hour, \
g_trunk_create_file_time_base.minute, \
g_trunk_create_file_interval, \
(int)(g_trunk_create_file_space_threshold / \
(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying, \
g_trunk_init_reload_from_binlog, \
g_trunk_compress_binlog_min_interval, \
g_store_slave_file_use_link);
if (g_use_storage_id && *g_sync_src_id != '\0' && \
!fdfs_is_server_id_valid(g_sync_src_id))
{
if ((result=storage_convert_src_server_id()) == 0)
{
storage_write_to_sync_ini_file();
}
else
{
if (result == ENOENT)
{
*g_sync_src_id = '\0';
storage_write_to_sync_ini_file();
}
else
{
return result;
}
}
}
return 0;
}
static int storage_check_and_make_data_dirs()
{
int result;
int i;
char data_path[MAX_PATH_SIZE];
char full_filename[MAX_PATH_SIZE];
bool pathCreated;
snprintf(data_path, sizeof(data_path), "%s/data", \
g_fdfs_base_path);
snprintf(full_filename, sizeof(full_filename), "%s/%s", \
data_path, DATA_DIR_INITED_FILENAME);
if (fileExists(full_filename))
{
IniContext iniContext;
char *pValue;
if ((result=iniLoadFromFile(full_filename, &iniContext)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"load from file \"%s/%s\" fail, " \
"error code: %d", \
__LINE__, data_path, \
full_filename, result);
return result;
}
pValue = iniGetStrValue(NULL, INIT_ITEM_STORAGE_JOIN_TIME, \
&iniContext);
if (pValue == NULL)
{
iniFreeContext(&iniContext);
logError("file: "__FILE__", line: %d, " \
"in file \"%s/%s\", item \"%s\" not exists", \
__LINE__, data_path, full_filename, \
INIT_ITEM_STORAGE_JOIN_TIME);
return ENOENT;
}
g_storage_join_time = atoi(pValue);
pValue = iniGetStrValue(NULL, INIT_ITEM_SYNC_OLD_DONE, \
&iniContext);
if (pValue == NULL)
{
iniFreeContext(&iniContext);
logError("file: "__FILE__", line: %d, " \
"in file \"%s/%s\", item \"%s\" not exists", \
__LINE__, data_path, full_filename, \
INIT_ITEM_SYNC_OLD_DONE);
return ENOENT;
}
g_sync_old_done = atoi(pValue);
pValue = iniGetStrValue(NULL, INIT_ITEM_SYNC_SRC_SERVER, \
&iniContext);
if (pValue == NULL)
{
iniFreeContext(&iniContext);
logError("file: "__FILE__", line: %d, " \
"in file \"%s/%s\", item \"%s\" not exists", \
__LINE__, data_path, full_filename, \
INIT_ITEM_SYNC_SRC_SERVER);
return ENOENT;
}
snprintf(g_sync_src_ip_addr, sizeof(g_sync_src_ip_addr), \
"%s", pValue);
g_sync_until_timestamp = iniGetIntValue(NULL, \
INIT_ITEM_SYNC_UNTIL_TIMESTAMP, \
&iniContext, 0);
pValue = iniGetStrValue(NULL, INIT_ITEM_LAST_IP_ADDRESS, \
&iniContext);
if (pValue != NULL)
{
snprintf(g_last_storage_ip, sizeof(g_last_storage_ip), \
"%s", pValue);
}
pValue = iniGetStrValue(NULL, INIT_ITEM_LAST_SERVER_PORT, \
&iniContext);
if (pValue != NULL)
{
g_last_server_port = atoi(pValue);
}
pValue = iniGetStrValue(NULL, INIT_ITEM_LAST_HTTP_PORT, \
&iniContext);
if (pValue != NULL)
{
g_last_http_port = atoi(pValue);
}
g_current_trunk_file_id = iniGetIntValue(NULL, \
INIT_ITEM_CURRENT_TRUNK_FILE_ID, &iniContext, 0);
iniFreeContext(&iniContext);
if (g_last_server_port == 0 || g_last_http_port == 0)
{
if (g_last_server_port == 0)
{
g_last_server_port = g_server_port;
}
if (g_last_http_port == 0)
{
g_last_http_port = g_http_port;
}
if ((result=storage_write_to_sync_ini_file()) != 0)
{
return result;
}
}
/*
printf("g_sync_old_done = %d\n", g_sync_old_done);
printf("g_sync_src_ip_addr = %s\n", g_sync_src_ip_addr);
printf("g_sync_until_timestamp = %d\n", g_sync_until_timestamp);
printf("g_last_storage_ip = %s\n", g_last_storage_ip);
printf("g_last_server_port = %d\n", g_last_server_port);
printf("g_last_http_port = %d\n", g_last_http_port);
printf("g_current_trunk_file_id = %d\n", g_current_trunk_file_id);
*/
}
else
{
if (!fileExists(data_path))
{
if (mkdir(data_path, 0755) != 0)
{
logError("file: "__FILE__", line: %d, " \
"mkdir \"%s\" fail, " \
"errno: %d, error info: %s", \
__LINE__, data_path, \
errno, STRERROR(errno));
return errno != 0 ? errno : EPERM;
}
STORAGE_CHOWN(data_path, geteuid(), getegid())
}
g_last_server_port = g_server_port;
g_last_http_port = g_http_port;
g_storage_join_time = time(NULL);
if ((result=storage_write_to_sync_ini_file()) != 0)
{
return result;
}
}
for (i=0; i<g_fdfs_path_count; i++)
{
if ((result=storage_make_data_dirs(g_fdfs_store_paths[i], \
&pathCreated)) != 0)
{
return result;
}
if (g_sync_old_done && pathCreated) //repair damaged disk
{
if ((result=storage_disk_recovery_start(i)) != 0)
{
return result;
}
}
result = storage_disk_recovery_restore(g_fdfs_store_paths[i]);
if (result == EAGAIN) //need to re-fetch binlog
{
if ((result=storage_disk_recovery_start(i)) != 0)
{
return result;
}
result=storage_disk_recovery_restore(g_fdfs_store_paths[i]);
}
if (result != 0)
{
return result;
}
}
return 0;
}
static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo)
{
char buff[32];
int i;
int result;
int filename_len;
char short_filename[64];
char full_filename[MAX_PATH_SIZE];
int store_path_index;
int sub_path_high;
int sub_path_low;
store_path_index = g_store_path_index;
if (g_store_path_mode == FDFS_STORE_PATH_LOAD_BALANCE)
{
if (store_path_index < 0)
{
return ENOSPC;
}
}
else
{
if (store_path_index >= g_fdfs_store_paths.count)
{
store_path_index = 0;
}
if (!storage_check_reserved_space_path(g_path_space_list \
[store_path_index].total_mb, g_path_space_list \
[store_path_index].free_mb, g_avg_storage_reserved_mb))
{
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if (storage_check_reserved_space_path( \
g_path_space_list[i].total_mb, \
g_path_space_list[i].free_mb, \
g_avg_storage_reserved_mb))
{
store_path_index = i;
g_store_path_index = i;
break;
}
}
if (i == g_fdfs_store_paths.count)
{
return ENOSPC;
}
}
g_store_path_index++;
if (g_store_path_index >= g_fdfs_store_paths.count)
{
g_store_path_index = 0;
}
}
pTrunkInfo->path.store_path_index = store_path_index;
while (1)
{
pthread_mutex_lock(&trunk_file_lock);
pTrunkInfo->file.id = ++g_current_trunk_file_id;
result = storage_write_to_sync_ini_file();
pthread_mutex_unlock(&trunk_file_lock);
if (result != 0)
{
return result;
}
int2buff(pTrunkInfo->file.id, buff);
base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(int), \
short_filename, &filename_len, false);
storage_get_store_path(short_filename, filename_len, \
&sub_path_high, &sub_path_low);
pTrunkInfo->path.sub_path_high = sub_path_high;
pTrunkInfo->path.sub_path_low = sub_path_low;
trunk_get_full_filename(pTrunkInfo, full_filename, \
sizeof(full_filename));
if (!fileExists(full_filename))
{
break;
}
}
if ((result=trunk_init_file(full_filename)) != 0)
{
return result;
}
return 0;
}
