support compress error log and access log

HISTORY

@@ -1,4 +1,9 @@
 
+Version 6.04  2019-11-30
+ * storage_report_ip_changed ignore result EEXIST
+ * use get_gzip_command_filename from libfastcommon v1.42
+ * support compress error log and access log
+
 Version 6.03  2019-11-20
  * dual IPs support two different types of inner (intranet) IPs
  * storage server request tracker server to change it's status

common/fdfs_global.c

@@ -23,7 +23,7 @@
 int g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT;
 int g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT;
 char g_fdfs_base_path[MAX_PATH_SIZE] = {'/', 't', 'm', 'p', '\0'};
-Version g_fdfs_version = {6, 3};
+Version g_fdfs_version = {6, 4};
 bool g_use_connection_pool = false;
 ConnectionPool g_connection_pool;
 int g_connection_pool_max_idle_time = 3600;

conf/storage.conf

@@ -255,6 +255,11 @@ rotate_access_log = false
 # since V4.00
 access_log_rotate_time=00:00
 
+# if compress the old access log by gzip
+# default value is false
+# since V6.04
+compress_old_access_log = false
+
 # if rotate the error log every day
 # default value is false
 # since V4.02
@@ -266,6 +271,11 @@ rotate_error_log = false
 # since V4.02
 error_log_rotate_time=00:00
 
+# if compress the old error log by gzip
+# default value is false
+# since V6.04
+compress_old_error_log = false
+
 # rotate access log when the log file exceeds this size
 # 0 means never rotates log file by log file size
 # default value is 0

conf/tracker.conf

@@ -240,7 +240,12 @@ rotate_error_log = false
 # Hour from 0 to 23, Minute from 0 to 59
 # default value is 00:00
 # since V4.02
-error_log_rotate_time=00:00
+error_log_rotate_time = 00:00
+
+# if compress the old error log by gzip
+# default value is false
+# since V6.04
+compress_old_error_log = false
 
 # rotate error log when the log file exceeds this size
 # 0 means never rotates log file by log file size

storage/storage_func.c

@@ -1994,6 +1994,23 @@ int storage_func_init(const char *filename, \
 
 		g_rotate_error_log = iniGetBoolValue(NULL, "rotate_error_log",\
 					&iniContext, false);
+		g_compress_old_access_log = iniGetBoolValue(NULL, "compress_old_access_log",
+					&iniContext, false);
+		g_compress_old_error_log = iniGetBoolValue(NULL, "compress_old_error_log",
+					&iniContext, false);
+
+        if (g_compress_old_error_log)
+        {
+            log_set_compress_log_flags(LOG_COMPRESS_FLAGS_ENABLED |
+                    LOG_COMPRESS_FLAGS_NEW_THREAD);
+        }
+		if (g_use_access_log && g_compress_old_access_log)
+        {
+            log_set_compress_log_flags_ex(&g_access_log_context,
+                    LOG_COMPRESS_FLAGS_ENABLED |
+                    LOG_COMPRESS_FLAGS_NEW_THREAD);
+        }
+
 		if ((result=get_time_item_from_conf(&iniContext, \
 			"error_log_rotate_time", &g_error_log_rotate_time, \
 			0, 0)) != 0)
@@ -2124,8 +2141,10 @@ int storage_func_init(const char *filename, \
 			"HTTP server port=%d, domain name=%s, " \
 			"use_access_log=%d, rotate_access_log=%d, " \
 			"access_log_rotate_time=%02d:%02d, " \
+            "compress_old_access_log=%d, " \
 			"rotate_error_log=%d, " \
 			"error_log_rotate_time=%02d:%02d, " \
+            "compress_old_error_log=%d, " \
 			"rotate_access_log_size=%"PRId64", " \
 			"rotate_error_log_size=%"PRId64", " \
 			"log_file_keep_days=%d, " \
@@ -2163,9 +2182,9 @@ int storage_func_init(const char *filename, \
 			g_key_namespace, g_keep_alive, \
 			g_http_port, g_http_domain, g_use_access_log, \
 			g_rotate_access_log, g_access_log_rotate_time.hour, \
-			g_access_log_rotate_time.minute, \
+			g_access_log_rotate_time.minute, g_compress_old_access_log, \
 			g_rotate_error_log, g_error_log_rotate_time.hour, \
-			g_error_log_rotate_time.minute, \
+			g_error_log_rotate_time.minute, g_compress_old_error_log, \
 			g_access_log_context.rotate_size, \
 			g_log_context.rotate_size, g_log_file_keep_days, \
 			g_file_sync_skip_invalid_record, \

storage/storage_global.c

@@ -79,6 +79,8 @@ in_addr_t g_server_id_in_filename = 0;
 bool g_use_access_log = false;    //if log to access log
 bool g_rotate_access_log = false; //if rotate the access log every day
 bool g_rotate_error_log = false;  //if rotate the error log every day
+bool g_compress_old_access_log = false; //if compress the old access log
+bool g_compress_old_error_log = false;  //if compress the old error log
 bool g_use_storage_id = false;    //identify storage by ID instead of IP address
 byte g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS; //id type of the storage server in the filename
 bool g_store_slave_file_use_link = false; //if store slave file use symbol link

storage/storage_global.h

@@ -128,6 +128,8 @@ extern byte g_id_type_in_filename; //id type of the storage server in the filena
 extern bool g_use_access_log;  //if log to access log
 extern bool g_rotate_access_log;  //if rotate the access log every day
 extern bool g_rotate_error_log;  //if rotate the error log every day
+extern bool g_compress_old_access_log; //if compress the old access log
+extern bool g_compress_old_error_log;  //if compress the old error log
 
 extern TimeInfo g_access_log_rotate_time; //rotate access log time base
 extern TimeInfo g_error_log_rotate_time;  //rotate error log time base

storage/storage_sync.c

@@ -1792,7 +1792,8 @@ static int uncompress_binlog_file(StorageBinLogReader *pReader,
     logInfo("file: "__FILE__", line: %d, "
             "try to uncompress binlog %s",
             __LINE__, gzip_filename);
-    snprintf(command, sizeof(command), "gzip -d %s 2>&1", gzip_filename);
+    snprintf(command, sizeof(command), "%s -d %s 2>&1",
+            get_gzip_command_filename(), gzip_filename);
     result = getExecResult(command, output, sizeof(output));
     unlink(flag_filename);
     if (result != 0)
@@ -1873,7 +1874,8 @@ static int compress_binlog_file(const char *filename)
             "try to compress binlog %s",
             __LINE__, filename);
 
-    snprintf(command, sizeof(command), "gzip %s 2>&1", filename);
+    snprintf(command, sizeof(command), "%s %s 2>&1",
+            get_gzip_command_filename(), filename);
     result = getExecResult(command, output, sizeof(output));
     unlink(flag_filename);
     if (result != 0)

tracker/tracker_func.c

@@ -579,8 +579,16 @@ int tracker_load_from_conf_file(const char *filename, \
 			return result;
 		}
 
-		g_rotate_error_log = iniGetBoolValue(NULL, "rotate_error_log",\
+		g_rotate_error_log = iniGetBoolValue(NULL, "rotate_error_log",
 					&iniContext, false);
+        g_compress_old_error_log = iniGetBoolValue(NULL, "compress_old_error_log",
+                &iniContext, false);
+        if (g_compress_old_error_log)
+        {
+            log_set_compress_log_flags(LOG_COMPRESS_FLAGS_ENABLED |
+                    LOG_COMPRESS_FLAGS_NEW_THREAD);
+        }
+
 		if ((result=get_time_item_from_conf(&iniContext, \
 			"error_log_rotate_time", &g_error_log_rotate_time, \
 			0, 0)) != 0)
@@ -744,6 +752,7 @@ int tracker_load_from_conf_file(const char *filename, \
 			"storage_id/ip_count=%d / %d, " \
 			"rotate_error_log=%d, " \
 			"error_log_rotate_time=%02d:%02d, " \
+            "compress_old_error_log=%d, " \
 			"rotate_error_log_size=%"PRId64", " \
 			"log_file_keep_days=%d, " \
 			"store_slave_file_use_link=%d, " \
@@ -780,7 +789,7 @@ int tracker_load_from_conf_file(const char *filename, \
 			FDFS_ID_TYPE_SERVER_ID ? "id" : "ip", \
             g_storage_ids_by_id.count, g_storage_ids_by_ip.count, \
 			g_rotate_error_log, g_error_log_rotate_time.hour, \
-			g_error_log_rotate_time.minute, \
+			g_error_log_rotate_time.minute, g_compress_old_error_log, \
 			g_log_context.rotate_size, g_log_file_keep_days,
 			g_store_slave_file_use_link, \
 			g_use_connection_pool, g_connection_pool_max_idle_time);

tracker/tracker_global.c

@@ -39,6 +39,7 @@ bool g_storage_ip_changed_auto_adjust = true;
 bool g_use_storage_id = false;  //if use storage ID instead of IP address
 byte g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS; //id type of the storage server in the filename
 bool g_rotate_error_log = false;  //if rotate the error log every day
+bool g_compress_old_error_log = false;  //if compress the old error log
 TimeInfo g_error_log_rotate_time  = {0, 0, 0}; //rotate error log time base
 
 int g_thread_stack_size = 64 * 1024;

tracker/tracker_global.h

@@ -62,8 +62,9 @@ extern char g_run_by_user[32];
 extern bool g_storage_ip_changed_auto_adjust;
 extern bool g_use_storage_id;  //identify storage by ID instead of IP address
 extern byte g_id_type_in_filename; //id type of the storage server in the filename
-extern bool g_rotate_error_log;  //if rotate the error log every day
+extern bool g_rotate_error_log;    //if rotate the error log every day
-extern TimeInfo g_error_log_rotate_time;  //rotate error log time base
+extern bool g_compress_old_error_log;    //if compress the old error log
+extern TimeInfo g_error_log_rotate_time; //rotate error log time base
 
 extern int g_thread_stack_size;
 extern int g_storage_sync_file_max_delay;