support multi ip for storage server (storage part)

storage/storage_dump.c

@@ -213,7 +213,8 @@ static int fdfs_dump_global_vars(char *buff, const int buffSize)
 		, g_store_path_index
 		, g_current_trunk_file_id
 		, g_trunk_sync_thread_count
-		, g_trunk_server.ip_addr, g_trunk_server.port
+		, g_trunk_server.connections[0].ip_addr
+        , g_trunk_server.connections[0].port
 		, g_trunk_total_free_space
 		, g_use_connection_pool
 		, g_connection_pool_max_idle_time

storage/storage_sync_func.c

@@ -38,21 +38,30 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
     int result;
     int i;
     FDFSMultiIP ip_addrs;
+    FDFSMultiIP *multi_ip;
 
+    multi_ip = NULL;
     if (g_use_storage_id)
     {
         FDFSStorageIdInfo *idInfo;
         idInfo = fdfs_get_storage_by_id(pStorage->id);
         if (idInfo == NULL)
         {
-            logError("file: "__FILE__", line: %d, "
+            logWarning("file: "__FILE__", line: %d, "
                     "storage server id: %s not exist "
-                    "in storage_ids.conf, storage ip: %s",
+                    "in storage_ids.conf from tracker server, "
-                    __LINE__, pStorage->id, pStorage->ip_addr);
+                    "storage ip: %s", __LINE__, pStorage->id,
-            sleep(5);
+                    pStorage->ip_addr);
-            return;
         }
-        ip_addrs = idInfo->ip_addrs;
+        else
+        {
+            multi_ip = &idInfo->ip_addrs;
+        }
+    }
+
+    if (multi_ip != NULL)
+    {
+        ip_addrs = *multi_ip;
     }
     else
     {
@@ -61,6 +70,7 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
         strcpy(ip_addrs.ips[0], pStorage->ip_addr);
     }
 
+    conn->sock = -1;
     nContinuousFail = 0;
     memset(previousCodes, 0, sizeof(previousCodes));
     memset(conn_results, 0, sizeof(conn_results));
@@ -121,7 +131,7 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
             conn->sock = -1;
         }
 
-        if (!g_continue_flag)
+        if (conn->sock >= 0 || !g_continue_flag)
         {
             break;
         }

storage/tracker_client_thread.c

@@ -1124,6 +1124,35 @@ static void get_tracker_leader()
     }
 }
 
+static void set_trunk_server(const char *ip_addr, const int port)
+{
+    if (g_use_storage_id)
+    {
+        FDFSStorageIdInfo *idInfo;
+        idInfo = fdfs_get_storage_id_by_ip(
+                g_group_name, ip_addr);
+        if (idInfo == NULL)
+        {
+            logWarning("file: "__FILE__", line: %d, "
+                    "storage server ip: %s not exist "
+                    "in storage_ids.conf from tracker server",
+                    __LINE__, ip_addr);
+
+            fdfs_set_server_info(&g_trunk_server,
+                    ip_addr, port);
+        }
+        else
+        {
+            fdfs_set_server_info_ex(&g_trunk_server,
+                    &idInfo->ip_addrs, port);
+        }
+    }
+    else
+    {
+        fdfs_set_server_info(&g_trunk_server, ip_addr, port);
+    }
+}
+
 static int tracker_check_response(ConnectionInfo *pTrackerServer, \
 	bool *bServerPortChanged)
 {
@@ -1302,26 +1331,26 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
 		}
 		else
 		{
-		memcpy(g_trunk_server.ip_addr, pBriefServers->ip_addr, \
+        int port;
-			IP_ADDRESS_SIZE - 1);
+
-		*(g_trunk_server.ip_addr + (IP_ADDRESS_SIZE - 1)) = '\0';
+        pBriefServers->ip_addr[IP_ADDRESS_SIZE - 1] = '\0';
-		g_trunk_server.port = buff2int(pBriefServers->port);
+        port = buff2int(pBriefServers->port);
-		if (is_local_host_ip(g_trunk_server.ip_addr) && \
+        set_trunk_server(pBriefServers->ip_addr, port);
-			g_trunk_server.port == g_server_port)
+		if (is_local_host_ip(pBriefServers->ip_addr) &&
+			port == g_server_port)
 		{
 			if (g_if_trunker_self)
 			{
-			logWarning("file: "__FILE__", line: %d, " \
+			logWarning("file: "__FILE__", line: %d, "
-				"I am already the trunk server %s:%d, " \
+				"I am already the trunk server %s:%d, "
-				"may be the tracker server restart", \
+				"may be the tracker server restart",
-				__LINE__, g_trunk_server.ip_addr, \
+				__LINE__, pBriefServers->ip_addr, port);
-				g_trunk_server.port);
 			}
 			else
 			{
-			logInfo("file: "__FILE__", line: %d, " \
+			logInfo("file: "__FILE__", line: %d, "
-				"I am the the trunk server %s:%d", __LINE__, \
+				"I am the the trunk server %s:%d", __LINE__,
-				g_trunk_server.ip_addr, g_trunk_server.port);
+				pBriefServers->ip_addr, port);
 
 			tracker_fetch_trunk_fid(pTrackerServer);
 			g_if_trunker_self = true;
@@ -1331,7 +1360,7 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
 				return result;
 			}
 
-			if (g_trunk_create_file_advance && \
+			if (g_trunk_create_file_advance &&
 				g_trunk_create_file_interval > 0)
 			{
 			ScheduleArray scheduleArray;
@@ -1353,9 +1382,10 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
 		}
 		else
 		{
-			logInfo("file: "__FILE__", line: %d, " \
+			logInfo("file: "__FILE__", line: %d, "
-				"the trunk server is %s:%d", __LINE__, \
+				"the trunk server is %s:%d", __LINE__,
-				g_trunk_server.ip_addr, g_trunk_server.port);
+				g_trunk_server.connections[0].ip_addr,
+                g_trunk_server.connections[0].port);
 
 			if (g_if_trunker_self)
 			{
@@ -1364,8 +1394,8 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
 				logWarning("file: "__FILE__", line: %d, " \
 					"I am the old trunk server, " \
 					"the new trunk server is %s:%d", \
-					__LINE__, g_trunk_server.ip_addr, \
+					__LINE__, g_trunk_server.connections[0].ip_addr, \
-					g_trunk_server.port);
+					g_trunk_server.connections[0].port);
 
 				tracker_report_trunk_fid(pTrackerServer);
 				g_if_trunker_self = false;

storage/trunk_mgr/trunk_client.c

@@ -93,11 +93,42 @@ static int trunk_client_trunk_do_alloc_space(ConnectionInfo *pTrunkServer, \
 	return 0;
 }
 
+static int trunk_client_connect_trunk_server(TrackerServerInfo *trunk_server,
+        ConnectionInfo **conn, const char *prompt)
+{
+	int result;
+
+	if (g_trunk_server.count == 0)
+	{
+		logError("file: "__FILE__", line: %d, "
+			"no trunk server", __LINE__);
+		return EAGAIN;
+	}
+
+	memcpy(trunk_server, &g_trunk_server, sizeof(TrackerServerInfo));
+	if ((*conn=tracker_connect_server(trunk_server, &result)) == NULL)
+	{
+		logError("file: "__FILE__", line: %d, "
+			"%s because connect to trunk "
+			"server %s:%d fail, errno: %d", __LINE__,
+			prompt, trunk_server->connections[0].ip_addr,
+            trunk_server->connections[0].port, result);
+		return result;
+	}
+
+    if (g_trunk_server.index != trunk_server->index)
+    {
+        g_trunk_server.index = trunk_server->index;
+    }
+
+    return 0;
+}
+
 int trunk_client_trunk_alloc_space(const int file_size, \
 		FDFSTrunkFullInfo *pTrunkInfo)
 {
 	int result;
-	ConnectionInfo trunk_server;
+	TrackerServerInfo trunk_server;
 	ConnectionInfo *pTrunkServer;
 
 	if (g_if_trunker_self)
@@ -105,20 +136,9 @@ int trunk_client_trunk_alloc_space(const int file_size, \
 		return trunk_alloc_space(file_size, pTrunkInfo);
 	}
 
-	if (*(g_trunk_server.ip_addr) == '\0')
+    if ((result=trunk_client_connect_trunk_server(&trunk_server,
+                    &pTrunkServer, "can't alloc trunk space")) != 0)
     {
-		logError("file: "__FILE__", line: %d, " \
-			"no trunk server", __LINE__);
-		return EAGAIN;
-	}
-
-	memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo));
-	if ((pTrunkServer=tracker_make_connection(&trunk_server, &result)) == NULL)
-	{
-		logError("file: "__FILE__", line: %d, " \
-			"can't alloc trunk space because connect to trunk " \
-			"server %s:%d fail, errno: %d", __LINE__, \
-			trunk_server.ip_addr, trunk_server.port, result);
         return result;
     }
 
@@ -202,7 +222,7 @@ int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \
 		const int status)
 {
 	int result;
-	ConnectionInfo trunk_server;
+	TrackerServerInfo trunk_server;
 	ConnectionInfo *pTrunkServer;
 
 	if (g_if_trunker_self)
@@ -210,18 +230,9 @@ int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \
 		return trunk_alloc_confirm(pTrunkInfo, status);
 	}
 
-	if (*(g_trunk_server.ip_addr) == '\0')
+    if ((result=trunk_client_connect_trunk_server(&trunk_server,
+                    &pTrunkServer, "trunk alloc confirm fail")) != 0)
     {
-		return EAGAIN;
-	}
-
-	memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo));
-	if ((pTrunkServer=tracker_make_connection(&trunk_server, &result)) == NULL)
-	{
-		logError("file: "__FILE__", line: %d, " \
-			"trunk alloc confirm fail because connect to trunk " \
-			"server %s:%d fail, errno: %d", __LINE__, \
-			trunk_server.ip_addr, trunk_server.port, result);
         return result;
     }
 
@@ -235,7 +246,7 @@ int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \
 int trunk_client_trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo)
 {
 	int result;
-	ConnectionInfo trunk_server;
+	TrackerServerInfo trunk_server;
 	ConnectionInfo *pTrunkServer;
 
 	if (g_if_trunker_self)
@@ -243,18 +254,9 @@ int trunk_client_trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo)
 		return trunk_free_space(pTrunkInfo, true);
 	}
 
-	if (*(g_trunk_server.ip_addr) == '\0')
+    if ((result=trunk_client_connect_trunk_server(&trunk_server,
+                    &pTrunkServer, "free trunk space fail")) != 0)
     {
-		return EAGAIN;
-	}
-
-	memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo));
-	if ((pTrunkServer=tracker_make_connection(&trunk_server, &result)) == NULL)
-	{
-		logError("file: "__FILE__", line: %d, " \
-			"free trunk space fail because connect to trunk " \
-			"server %s:%d fail, errno: %d", __LINE__, \
-			trunk_server.ip_addr, trunk_server.port, result);
         return result;
     }
 

storage/trunk_mgr/trunk_mem.c

@@ -54,7 +54,7 @@ int g_current_trunk_file_id = 0;
 TimeInfo g_trunk_create_file_time_base = {0, 0};
 int g_trunk_create_file_interval = 86400;
 int g_trunk_compress_binlog_min_interval = 0;
-ConnectionInfo g_trunk_server = {-1, 0};
+TrackerServerInfo g_trunk_server = {0, 0};
 bool g_if_use_trunk_file = false;
 bool g_if_trunker_self = false;
 bool g_trunk_create_file_advance = false;
@@ -176,9 +176,7 @@ int storage_trunk_init()
 	logDebug("file: "__FILE__", line: %d, " \
 		"storage trunk init ...", __LINE__);
 
-	g_trunk_server.sock = -1;
+    memset(&g_trunk_server, 0, sizeof(g_trunk_server));
-	g_trunk_server.port = g_server_port;
-
 	if ((result=init_pthread_lock(&trunk_file_lock)) != 0)
 	{
 		logError("file: "__FILE__", line: %d, " \

storage/trunk_mgr/trunk_mem.h

@@ -37,7 +37,7 @@ extern int g_current_trunk_file_id;  //current trunk file id
 extern TimeInfo g_trunk_create_file_time_base;
 extern int g_trunk_create_file_interval;
 extern int g_trunk_compress_binlog_min_interval;
-extern ConnectionInfo g_trunk_server;  //the trunk server
+extern TrackerServerInfo g_trunk_server;  //the trunk server
 extern bool g_if_use_trunk_file;   //if use trunk file
 extern bool g_trunk_create_file_advance;
 extern bool g_trunk_init_check_occupying;

tracker/fdfs_shared_func.c

@@ -660,3 +660,25 @@ void fdfs_set_multi_ip_index(FDFSMultiIP *multi_ip, const char *target_ip)
         }
     }
 }
+
+void fdfs_set_server_info(TrackerServerInfo *pServer,
+        const char *ip_addr, const int port)
+{
+    pServer->count = 1;
+    pServer->index = 0;
+    conn_pool_set_server_info(pServer->connections + 0, ip_addr, port);
+}
+
+void fdfs_set_server_info_ex(TrackerServerInfo *pServer,
+        const FDFSMultiIP *ip_addrs, const int port)
+{
+    int i;
+
+    pServer->count = ip_addrs->count;
+    pServer->index = 0;
+    for (i=0; i<ip_addrs->count; i++)
+    {
+        conn_pool_set_server_info(pServer->connections + i,
+                ip_addrs->ips[i], port);
+    }
+}

tracker/fdfs_shared_func.h

@@ -124,6 +124,12 @@ const char *fdfs_get_ipaddr_by_client_ip(const FDFSMultiIP *ip_addrs,
 
 void fdfs_set_multi_ip_index(FDFSMultiIP *multi_ip, const char *target_ip);
 
+void fdfs_set_server_info(TrackerServerInfo *pServer,
+        const char *ip_addr, const int port);
+
+void fdfs_set_server_info_ex(TrackerServerInfo *pServer,
+        const FDFSMultiIP *ip_addrs, const int port);
+
 #ifdef __cplusplus
 }
 #endif

tracker/tracker_proto.c

@@ -544,14 +544,14 @@ void tracker_disconnect_server(TrackerServerInfo *pServerInfo)
 
     if (pServerInfo->count == 1)
     {
-		conn_pool_disconnect_server(pServerInfo->connections);
+        tracker_close_connection_ex(pServerInfo->connections + 0, true);
         return;
     }
 
 	end = pServerInfo->connections + pServerInfo->count;
 	for (conn=pServerInfo->connections; conn<end; conn++)
     {
-		conn_pool_disconnect_server(conn);
+        tracker_close_connection_ex(conn, true);
     }
 }