diff --git a/HISTORY b/HISTORY index c77a6de..3807ec7 100644 --- a/HISTORY +++ b/HISTORY @@ -6,11 +6,19 @@ Version 6.05 2019-12-23 * trunk server support compress the trunk binlog periodically, the config items in tracker.conf: trunk_compress_binlog_interval and trunk_compress_binlog_time_base + * trunk binlog compression support transaction * support backup binlog file when truncate trunk binlog, the config item in tracker.conf: trunk_binlog_max_backups + * support alignment size for trunk space allocation + the config item in tracker.conf: trunk_alloc_alignment_size + * support merge free trunk spaces + the config item in tracker.conf: trunk_free_space_merge + + * support delete unused trunk files + the config item in tracker.conf: delete_unused_trunk_files Version 6.04 2019-12-05 * storage_report_ip_changed ignore result EEXIST diff --git a/conf/tracker.conf b/conf/tracker.conf index 26e677e..5ffdfe8 100644 --- a/conf/tracker.conf +++ b/conf/tracker.conf @@ -146,7 +146,7 @@ use_trunk_file = false # the min slot size, should <= 4KB # default value is 256 bytes # since V3.00 -slot_min_size = 256 +slot_min_size = 512 # the max slot size, should > slot_min_size # store the upload file to trunk file when it's size <= this value @@ -166,6 +166,11 @@ trunk_alloc_alignment_size = 512 # since V6.05 trunk_free_space_merge = true +# if delete / reclaim the unused trunk files +# default value is false +# since V6.05 +delete_unused_trunk_files = true + # the trunk file size, should >= 4MB # default value is 64MB # since V3.00 diff --git a/storage/storage_param_getter.c b/storage/storage_param_getter.c index 62ebd3c..ed05270 100644 --- a/storage/storage_param_getter.c +++ b/storage/storage_param_getter.c @@ -144,6 +144,8 @@ int storage_get_params_from_tracker() "trunk_init_reload_from_binlog", &iniContext, false); g_trunk_free_space_merge = iniGetBoolValue(NULL, "trunk_free_space_merge", &iniContext, false); + g_delete_unused_trunk_files = iniGetBoolValue(NULL, + "delete_unused_trunk_files", &iniContext, false); g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL, "trunk_compress_binlog_min_interval", &iniContext, 0); g_trunk_compress_binlog_interval = iniGetIntValue(NULL, @@ -206,6 +208,7 @@ int storage_get_params_from_tracker() "trunk_init_check_occupying=%d, " "trunk_init_reload_from_binlog=%d, " "trunk_free_space_merge=%d, " + "delete_unused_trunk_files=%d, " "trunk_compress_binlog_min_interval=%d, " "trunk_compress_binlog_interval=%d, " "trunk_compress_binlog_time_base=%02d:%02d, " @@ -228,6 +231,7 @@ int storage_get_params_from_tracker() (FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying, g_trunk_init_reload_from_binlog, g_trunk_free_space_merge, + g_delete_unused_trunk_files, g_trunk_compress_binlog_min_interval, g_trunk_compress_binlog_interval, g_trunk_compress_binlog_time_base.hour, diff --git a/storage/trunk_mgr/trunk_mem.c b/storage/trunk_mgr/trunk_mem.c index 2e49d8f..76ca2cf 100644 --- a/storage/trunk_mgr/trunk_mem.c +++ b/storage/trunk_mgr/trunk_mem.c @@ -65,6 +65,7 @@ bool g_trunk_create_file_advance = false; bool g_trunk_init_check_occupying = false; bool g_trunk_init_reload_from_binlog = false; bool g_trunk_free_space_merge = false; +bool g_delete_unused_trunk_files = false; int g_trunk_binlog_compress_stage = STORAGE_TRUNK_COMPRESS_STAGE_NONE; volatile int64_t g_trunk_total_free_space = 0; int64_t g_trunk_create_file_space_threshold = 0; @@ -542,17 +543,20 @@ typedef struct trunk_merge_stat } TrunkMergeStat; static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst, - FDFSTrunkFullInfo **ppLast, TrunkMergeStat *stat) + FDFSTrunkFullInfo **ppLast, TrunkMergeStat *merge_stat, + bool *bDeleted) { FDFSTrunkFullInfo **ppTrunkInfo; int append_size; + char full_filename[MAX_PATH_SIZE]; + struct stat file_stat; (*ppMergeFirst)->file.size = (*ppLast)->file.offset - (*ppMergeFirst)->file.offset + (*ppLast)->file.size; - stat->merge_count++; - stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1; - stat->merged_size += (*ppMergeFirst)->file.size; + merge_stat->merge_count++; + merge_stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1; + merge_stat->merged_size += (*ppMergeFirst)->file.size; append_size = 0; for (ppTrunkInfo=ppMergeFirst + 1; ppTrunkInfo<=ppLast; ppTrunkInfo++) @@ -563,6 +567,52 @@ static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst, __sync_add_and_fetch(&g_trunk_total_free_space, append_size); + do + { + if (!g_delete_unused_trunk_files) + { + break; + } + + if (!((*ppMergeFirst)->file.offset == 0 && + (*ppMergeFirst)->file.size >= g_trunk_file_size)) + { + break; + } + + trunk_get_full_filename(*ppMergeFirst, full_filename, + sizeof(full_filename)); + if (stat(full_filename, &file_stat) != 0) + { + logError("file: "__FILE__", line: %d, " + "stat trunk file %s fail, " + "errno: %d, error info: %s", __LINE__, + full_filename, errno, STRERROR(errno)); + break; + } + if ((*ppMergeFirst)->file.size != file_stat.st_size) + { + break; + } + + if (unlink(full_filename) != 0) + { + if (errno != ENOENT) + { + logError("file: "__FILE__", line: %d, " + "unlink trunk file %s fail, " + "errno: %d, error info: %s", __LINE__, + full_filename, errno, STRERROR(errno)); + break; + } + } + + logInfo("file: "__FILE__", line: %d, " + "delete unused trunk file: %s", + __LINE__, full_filename); + trunk_delete_space_ex(*ppMergeFirst, false, false); + *bDeleted = true; + } while (0); } static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs) @@ -574,6 +624,7 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs) TrunkMergeStat merge_stat; char comma_buff[32]; int result; + bool bDeleted; if (pCallbackArgs->trunk_array.count == 0) { @@ -602,25 +653,35 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs) continue; } + bDeleted = false; if (ppTrunkInfo - ppMergeFirst > 1) { - trunk_merge_spaces(ppMergeFirst, previous, &merge_stat); + trunk_merge_spaces(ppMergeFirst, previous, + &merge_stat, &bDeleted); } - if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0) + if (!bDeleted) { - return result; + if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0) + { + return result; + } } ppMergeFirst = previous = ppTrunkInfo; } + bDeleted = false; if (ppEnd - ppMergeFirst > 1) { - trunk_merge_spaces(ppMergeFirst, previous, &merge_stat); + trunk_merge_spaces(ppMergeFirst, previous, + &merge_stat, &bDeleted); } - if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0) + if (!bDeleted) { - return result; + if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0) + { + return result; + } } logInfo("file: "__FILE__", line: %d, " diff --git a/storage/trunk_mgr/trunk_mem.h b/storage/trunk_mgr/trunk_mem.h index fe65922..bed9650 100644 --- a/storage/trunk_mgr/trunk_mem.h +++ b/storage/trunk_mgr/trunk_mem.h @@ -58,6 +58,7 @@ extern bool g_trunk_create_file_advance; extern bool g_trunk_init_check_occupying; extern bool g_trunk_init_reload_from_binlog; extern bool g_trunk_free_space_merge; +extern bool g_delete_unused_trunk_files; extern int g_trunk_binlog_compress_stage; extern bool g_if_trunker_self; //if am i trunk server extern int64_t g_trunk_create_file_space_threshold; diff --git a/tracker/tracker_func.c b/tracker/tracker_func.c index 00d9824..be08a4c 100644 --- a/tracker/tracker_func.c +++ b/tracker/tracker_func.c @@ -601,6 +601,9 @@ int tracker_load_from_conf_file(const char *filename, \ g_trunk_free_space_merge = iniGetBoolValue(NULL, "trunk_free_space_merge", &iniContext, false); + g_delete_unused_trunk_files = iniGetBoolValue(NULL, + "delete_unused_trunk_files", &iniContext, false); + if ((result=tracker_load_storage_id_info( filename, &iniContext)) != 0) { @@ -779,6 +782,7 @@ int tracker_load_from_conf_file(const char *filename, \ "trunk_init_check_occupying=%d, " "trunk_init_reload_from_binlog=%d, " "trunk_free_space_merge=%d, " + "delete_unused_trunk_files=%d, " "trunk_compress_binlog_min_interval=%d, " "trunk_compress_binlog_interval=%d, " "trunk_compress_binlog_time_base=%02d:%02d, " @@ -823,6 +827,7 @@ int tracker_load_from_conf_file(const char *filename, \ (FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying, g_trunk_init_reload_from_binlog, g_trunk_free_space_merge, + g_delete_unused_trunk_files, g_trunk_compress_binlog_min_interval, g_trunk_compress_binlog_interval, g_trunk_compress_binlog_time_base.hour, diff --git a/tracker/tracker_global.c b/tracker/tracker_global.c index bd19da4..a02d530 100644 --- a/tracker/tracker_global.c +++ b/tracker/tracker_global.c @@ -52,6 +52,7 @@ bool g_trunk_create_file_advance = false; bool g_trunk_init_check_occupying = false; bool g_trunk_init_reload_from_binlog = false; bool g_trunk_free_space_merge = false; +bool g_delete_unused_trunk_files = false; int g_slot_min_size = 256; //slot min size, such as 256 bytes int g_slot_max_size = 16 * 1024 * 1024; //slot max size, such as 16MB int g_trunk_file_size = 64 * 1024 * 1024; //the trunk file size, such as 64MB diff --git a/tracker/tracker_global.h b/tracker/tracker_global.h index cfd489f..fc0580f 100644 --- a/tracker/tracker_global.h +++ b/tracker/tracker_global.h @@ -76,6 +76,7 @@ extern bool g_trunk_create_file_advance; extern bool g_trunk_init_check_occupying; extern bool g_trunk_init_reload_from_binlog; extern bool g_trunk_free_space_merge; +extern bool g_delete_unused_trunk_files; extern int g_slot_min_size; //slot min size, such as 256 bytes extern int g_slot_max_size; //slot max size, such as 16MB extern int g_trunk_file_size; //the trunk file size, such as 64MB diff --git a/tracker/tracker_service.c b/tracker/tracker_service.c index f98d8fb..063df82 100644 --- a/tracker/tracker_service.c +++ b/tracker/tracker_service.c @@ -665,12 +665,12 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask) if (pTask->length - sizeof(TrackerHeader) != 0) { - logError("file: "__FILE__", line: %d, " \ - "cmd=%d, client ip: %s, package size " \ - PKG_LEN_PRINTF_FORMAT" is not correct, " \ - "expect length = %d", __LINE__, \ - TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ, \ - pTask->client_ip, pTask->length - \ + logError("file: "__FILE__", line: %d, " + "cmd=%d, client ip: %s, package size " + PKG_LEN_PRINTF_FORMAT" is not correct, " + "expect length = %d", __LINE__, + TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ, + pTask->client_ip, pTask->length - (int)sizeof(TrackerHeader), 0); pTask->length = sizeof(TrackerHeader); @@ -696,6 +696,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask) "trunk_init_check_occupying=%d\n" "trunk_init_reload_from_binlog=%d\n" "trunk_free_space_merge=%d\n" + "delete_unused_trunk_files=%d\n" "trunk_compress_binlog_min_interval=%d\n" "trunk_compress_binlog_interval=%d\n" "trunk_compress_binlog_time_base=%02d:%02d\n" @@ -718,6 +719,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask) g_trunk_init_check_occupying, g_trunk_init_reload_from_binlog, g_trunk_free_space_merge, + g_delete_unused_trunk_files, g_trunk_compress_binlog_min_interval, g_trunk_compress_binlog_interval, g_trunk_compress_binlog_time_base.hour,