support delete unused trunk files

HISTORY

@@ -6,11 +6,19 @@ Version 6.05  2019-12-23
  * trunk server support compress the trunk binlog periodically,
    the config items in tracker.conf: trunk_compress_binlog_interval
    and trunk_compress_binlog_time_base
+
  * trunk binlog compression support transaction
  * support backup binlog file when truncate trunk binlog,
    the config item in tracker.conf: trunk_binlog_max_backups
+
  * support alignment size for trunk space allocation
+   the config item in tracker.conf: trunk_alloc_alignment_size 
+
  * support merge free trunk spaces
+   the config item in tracker.conf: trunk_free_space_merge
+
+ * support delete unused trunk files
+   the config item in tracker.conf: delete_unused_trunk_files
 
 Version 6.04  2019-12-05
  * storage_report_ip_changed ignore result EEXIST

conf/tracker.conf

@@ -146,7 +146,7 @@ use_trunk_file = false
 # the min slot size, should <= 4KB
 # default value is 256 bytes
 # since V3.00
-slot_min_size = 256
+slot_min_size = 512
 
 # the max slot size, should > slot_min_size
 # store the upload file to trunk file when it's size <=  this value
@@ -166,6 +166,11 @@ trunk_alloc_alignment_size = 512
 # since V6.05
 trunk_free_space_merge = true
 
+# if delete / reclaim the unused trunk files
+# default value is false
+# since V6.05
+delete_unused_trunk_files = true
+
 # the trunk file size, should >= 4MB
 # default value is 64MB
 # since V3.00

storage/storage_param_getter.c

@@ -144,6 +144,8 @@ int storage_get_params_from_tracker()
 			"trunk_init_reload_from_binlog", &iniContext, false);
 	g_trunk_free_space_merge = iniGetBoolValue(NULL,
 			"trunk_free_space_merge", &iniContext, false);
+	g_delete_unused_trunk_files = iniGetBoolValue(NULL,
+			"delete_unused_trunk_files", &iniContext, false);
 	g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL,
 			"trunk_compress_binlog_min_interval", &iniContext, 0);
 	g_trunk_compress_binlog_interval = iniGetIntValue(NULL,
@@ -206,6 +208,7 @@ int storage_get_params_from_tracker()
 		"trunk_init_check_occupying=%d, "
 		"trunk_init_reload_from_binlog=%d, "
 		"trunk_free_space_merge=%d, "
+		"delete_unused_trunk_files=%d, "
 		"trunk_compress_binlog_min_interval=%d, "
 		"trunk_compress_binlog_interval=%d, "
 		"trunk_compress_binlog_time_base=%02d:%02d, "
@@ -228,6 +231,7 @@ int storage_get_params_from_tracker()
 		(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying,
 		g_trunk_init_reload_from_binlog,
 		g_trunk_free_space_merge,
+		g_delete_unused_trunk_files,
 		g_trunk_compress_binlog_min_interval,
 		g_trunk_compress_binlog_interval,
         g_trunk_compress_binlog_time_base.hour,

storage/trunk_mgr/trunk_mem.c

@@ -65,6 +65,7 @@ bool g_trunk_create_file_advance = false;
 bool g_trunk_init_check_occupying = false;
 bool g_trunk_init_reload_from_binlog = false;
 bool g_trunk_free_space_merge = false;
+bool g_delete_unused_trunk_files = false;
 int g_trunk_binlog_compress_stage = STORAGE_TRUNK_COMPRESS_STAGE_NONE;
 volatile int64_t g_trunk_total_free_space = 0;
 int64_t g_trunk_create_file_space_threshold = 0;
@@ -542,17 +543,20 @@ typedef struct trunk_merge_stat
 } TrunkMergeStat;
 
 static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
-        FDFSTrunkFullInfo **ppLast, TrunkMergeStat *stat)
+        FDFSTrunkFullInfo **ppLast, TrunkMergeStat *merge_stat,
+        bool *bDeleted)
 {
 	FDFSTrunkFullInfo **ppTrunkInfo;
     int append_size;
+    char full_filename[MAX_PATH_SIZE];
+    struct stat file_stat;
 
     (*ppMergeFirst)->file.size = (*ppLast)->file.offset -
         (*ppMergeFirst)->file.offset + (*ppLast)->file.size;
 
-    stat->merge_count++;
+    merge_stat->merge_count++;
-    stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1;
+    merge_stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1;
-    stat->merged_size += (*ppMergeFirst)->file.size;
+    merge_stat->merged_size += (*ppMergeFirst)->file.size;
 
     append_size = 0;
     for (ppTrunkInfo=ppMergeFirst + 1; ppTrunkInfo<=ppLast; ppTrunkInfo++)
@@ -563,6 +567,52 @@ static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
 
     __sync_add_and_fetch(&g_trunk_total_free_space,
             append_size);
+    do
+    {
+        if (!g_delete_unused_trunk_files)
+        {
+            break;
+        }
+
+        if (!((*ppMergeFirst)->file.offset == 0 &&
+                    (*ppMergeFirst)->file.size >= g_trunk_file_size))
+        {
+            break;
+        }
+
+        trunk_get_full_filename(*ppMergeFirst, full_filename,
+                sizeof(full_filename));
+        if (stat(full_filename, &file_stat) != 0)
+        {
+            logError("file: "__FILE__", line: %d, "
+                    "stat trunk file %s fail, "
+                    "errno: %d, error info: %s", __LINE__,
+                    full_filename, errno, STRERROR(errno));
+            break;
+        }
+        if ((*ppMergeFirst)->file.size != file_stat.st_size)
+        {
+            break;
+        }
+
+        if (unlink(full_filename) != 0)
+        {
+            if (errno != ENOENT)
+            {
+                logError("file: "__FILE__", line: %d, "
+                        "unlink trunk file %s fail, "
+                        "errno: %d, error info: %s", __LINE__,
+                        full_filename, errno, STRERROR(errno));
+                break;
+            }
+        }
+
+        logInfo("file: "__FILE__", line: %d, "
+                "delete unused trunk file: %s",
+                __LINE__, full_filename);
+        trunk_delete_space_ex(*ppMergeFirst, false, false);
+        *bDeleted = true;
+    } while (0);
 }
 
 static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
@@ -574,6 +624,7 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
     TrunkMergeStat merge_stat;
     char comma_buff[32];
 	int result;
+    bool bDeleted;
 
     if (pCallbackArgs->trunk_array.count == 0)
     {
@@ -602,25 +653,35 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
             continue;
         }
 
+        bDeleted = false;
         if (ppTrunkInfo - ppMergeFirst > 1)
         {
-            trunk_merge_spaces(ppMergeFirst, previous, &merge_stat);
+            trunk_merge_spaces(ppMergeFirst, previous,
+                    &merge_stat, &bDeleted);
         }
-        if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
+        if (!bDeleted)
         {
-            return result;
+            if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
+            {
+                return result;
+            }
         }
 
         ppMergeFirst = previous = ppTrunkInfo;
 	}
 
+    bDeleted = false;
     if (ppEnd - ppMergeFirst > 1)
     {
-        trunk_merge_spaces(ppMergeFirst, previous, &merge_stat);
+        trunk_merge_spaces(ppMergeFirst, previous,
+                &merge_stat, &bDeleted);
     }
-    if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
+    if (!bDeleted)
     {
-        return result;
+        if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
+        {
+            return result;
+        }
     }
 
     logInfo("file: "__FILE__", line: %d, "

storage/trunk_mgr/trunk_mem.h

@@ -58,6 +58,7 @@ extern bool g_trunk_create_file_advance;
 extern bool g_trunk_init_check_occupying;
 extern bool g_trunk_init_reload_from_binlog;
 extern bool g_trunk_free_space_merge;
+extern bool g_delete_unused_trunk_files;
 extern int g_trunk_binlog_compress_stage;
 extern bool g_if_trunker_self;   //if am i trunk server
 extern int64_t g_trunk_create_file_space_threshold;

tracker/tracker_func.c

@@ -601,6 +601,9 @@ int tracker_load_from_conf_file(const char *filename, \
 		g_trunk_free_space_merge = iniGetBoolValue(NULL,
 			"trunk_free_space_merge", &iniContext, false);
 
+		g_delete_unused_trunk_files = iniGetBoolValue(NULL,
+			"delete_unused_trunk_files", &iniContext, false);
+
 		if ((result=tracker_load_storage_id_info(
 				filename, &iniContext)) != 0)
 		{
@@ -779,6 +782,7 @@ int tracker_load_from_conf_file(const char *filename, \
 			"trunk_init_check_occupying=%d, "
 			"trunk_init_reload_from_binlog=%d, "
 			"trunk_free_space_merge=%d, "
+			"delete_unused_trunk_files=%d, "
 			"trunk_compress_binlog_min_interval=%d, "
 			"trunk_compress_binlog_interval=%d, "
 			"trunk_compress_binlog_time_base=%02d:%02d, "
@@ -823,6 +827,7 @@ int tracker_load_from_conf_file(const char *filename, \
 			(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying,
 			g_trunk_init_reload_from_binlog,
             g_trunk_free_space_merge,
+            g_delete_unused_trunk_files,
 			g_trunk_compress_binlog_min_interval,
 			g_trunk_compress_binlog_interval,
             g_trunk_compress_binlog_time_base.hour,

tracker/tracker_global.c

@@ -52,6 +52,7 @@ bool g_trunk_create_file_advance = false;
 bool g_trunk_init_check_occupying = false;
 bool g_trunk_init_reload_from_binlog = false;
 bool g_trunk_free_space_merge = false;
+bool g_delete_unused_trunk_files  = false;
 int g_slot_min_size = 256;    //slot min size, such as 256 bytes
 int g_slot_max_size = 16 * 1024 * 1024;    //slot max size, such as 16MB
 int g_trunk_file_size = 64 * 1024 * 1024;  //the trunk file size, such as 64MB

tracker/tracker_global.h

@@ -76,6 +76,7 @@ extern bool g_trunk_create_file_advance;
 extern bool g_trunk_init_check_occupying;
 extern bool g_trunk_init_reload_from_binlog;
 extern bool g_trunk_free_space_merge;
+extern bool g_delete_unused_trunk_files;
 extern int g_slot_min_size;    //slot min size, such as 256 bytes
 extern int g_slot_max_size;    //slot max size, such as 16MB
 extern int g_trunk_file_size;  //the trunk file size, such as 64MB

tracker/tracker_service.c

@@ -665,12 +665,12 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
 
 	if (pTask->length - sizeof(TrackerHeader) != 0)
 	{
-		logError("file: "__FILE__", line: %d, " \
+		logError("file: "__FILE__", line: %d, "
-			"cmd=%d, client ip: %s, package size " \
+			"cmd=%d, client ip: %s, package size "
-			PKG_LEN_PRINTF_FORMAT" is not correct, " \
+			PKG_LEN_PRINTF_FORMAT" is not correct, "
-			"expect length = %d", __LINE__, \
+			"expect length = %d", __LINE__,
-			TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ, \
+			TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ,
-			pTask->client_ip, pTask->length - \
+			pTask->client_ip, pTask->length -
 			(int)sizeof(TrackerHeader), 0);
 
 		pTask->length = sizeof(TrackerHeader);
@@ -696,6 +696,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
             "trunk_init_check_occupying=%d\n"
             "trunk_init_reload_from_binlog=%d\n"
             "trunk_free_space_merge=%d\n"
+            "delete_unused_trunk_files=%d\n"
             "trunk_compress_binlog_min_interval=%d\n"
             "trunk_compress_binlog_interval=%d\n"
             "trunk_compress_binlog_time_base=%02d:%02d\n"
@@ -718,6 +719,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
         g_trunk_init_check_occupying,
         g_trunk_init_reload_from_binlog,
         g_trunk_free_space_merge,
+        g_delete_unused_trunk_files,
         g_trunk_compress_binlog_min_interval,
         g_trunk_compress_binlog_interval,
         g_trunk_compress_binlog_time_base.hour,