From ba527d41bbc8a6ed8a66ab8a1f9d6a5bb8e88893 Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Sat, 12 Oct 2019 10:41:32 +0800 Subject: [PATCH] support multi ip for storage server (storage part) --- storage/storage_dump.c | 3 +- storage/storage_sync_func.c | 24 +++++--- storage/tracker_client_thread.c | 70 ++++++++++++++++------- storage/trunk_mgr/trunk_client.c | 96 ++++++++++++++++---------------- storage/trunk_mgr/trunk_mem.c | 6 +- storage/trunk_mgr/trunk_mem.h | 2 +- tracker/fdfs_shared_func.c | 22 ++++++++ tracker/fdfs_shared_func.h | 6 ++ tracker/tracker_proto.c | 4 +- 9 files changed, 151 insertions(+), 82 deletions(-) diff --git a/storage/storage_dump.c b/storage/storage_dump.c index 32a62db..e04c0ab 100644 --- a/storage/storage_dump.c +++ b/storage/storage_dump.c @@ -213,7 +213,8 @@ static int fdfs_dump_global_vars(char *buff, const int buffSize) , g_store_path_index , g_current_trunk_file_id , g_trunk_sync_thread_count - , g_trunk_server.ip_addr, g_trunk_server.port + , g_trunk_server.connections[0].ip_addr + , g_trunk_server.connections[0].port , g_trunk_total_free_space , g_use_connection_pool , g_connection_pool_max_idle_time diff --git a/storage/storage_sync_func.c b/storage/storage_sync_func.c index 6f527e9..ec8b9d3 100644 --- a/storage/storage_sync_func.c +++ b/storage/storage_sync_func.c @@ -38,21 +38,30 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage, int result; int i; FDFSMultiIP ip_addrs; + FDFSMultiIP *multi_ip; + multi_ip = NULL; if (g_use_storage_id) { FDFSStorageIdInfo *idInfo; idInfo = fdfs_get_storage_by_id(pStorage->id); if (idInfo == NULL) { - logError("file: "__FILE__", line: %d, " + logWarning("file: "__FILE__", line: %d, " "storage server id: %s not exist " - "in storage_ids.conf, storage ip: %s", - __LINE__, pStorage->id, pStorage->ip_addr); - sleep(5); - return; + "in storage_ids.conf from tracker server, " + "storage ip: %s", __LINE__, pStorage->id, + pStorage->ip_addr); } - ip_addrs = idInfo->ip_addrs; + else + { + multi_ip = &idInfo->ip_addrs; + } + } + + if (multi_ip != NULL) + { + ip_addrs = *multi_ip; } else { @@ -61,6 +70,7 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage, strcpy(ip_addrs.ips[0], pStorage->ip_addr); } + conn->sock = -1; nContinuousFail = 0; memset(previousCodes, 0, sizeof(previousCodes)); memset(conn_results, 0, sizeof(conn_results)); @@ -121,7 +131,7 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage, conn->sock = -1; } - if (!g_continue_flag) + if (conn->sock >= 0 || !g_continue_flag) { break; } diff --git a/storage/tracker_client_thread.c b/storage/tracker_client_thread.c index 80ae75a..fa6ac66 100644 --- a/storage/tracker_client_thread.c +++ b/storage/tracker_client_thread.c @@ -1124,6 +1124,35 @@ static void get_tracker_leader() } } +static void set_trunk_server(const char *ip_addr, const int port) +{ + if (g_use_storage_id) + { + FDFSStorageIdInfo *idInfo; + idInfo = fdfs_get_storage_id_by_ip( + g_group_name, ip_addr); + if (idInfo == NULL) + { + logWarning("file: "__FILE__", line: %d, " + "storage server ip: %s not exist " + "in storage_ids.conf from tracker server", + __LINE__, ip_addr); + + fdfs_set_server_info(&g_trunk_server, + ip_addr, port); + } + else + { + fdfs_set_server_info_ex(&g_trunk_server, + &idInfo->ip_addrs, port); + } + } + else + { + fdfs_set_server_info(&g_trunk_server, ip_addr, port); + } +} + static int tracker_check_response(ConnectionInfo *pTrackerServer, \ bool *bServerPortChanged) { @@ -1302,26 +1331,26 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \ } else { - memcpy(g_trunk_server.ip_addr, pBriefServers->ip_addr, \ - IP_ADDRESS_SIZE - 1); - *(g_trunk_server.ip_addr + (IP_ADDRESS_SIZE - 1)) = '\0'; - g_trunk_server.port = buff2int(pBriefServers->port); - if (is_local_host_ip(g_trunk_server.ip_addr) && \ - g_trunk_server.port == g_server_port) + int port; + + pBriefServers->ip_addr[IP_ADDRESS_SIZE - 1] = '\0'; + port = buff2int(pBriefServers->port); + set_trunk_server(pBriefServers->ip_addr, port); + if (is_local_host_ip(pBriefServers->ip_addr) && + port == g_server_port) { if (g_if_trunker_self) { - logWarning("file: "__FILE__", line: %d, " \ - "I am already the trunk server %s:%d, " \ - "may be the tracker server restart", \ - __LINE__, g_trunk_server.ip_addr, \ - g_trunk_server.port); + logWarning("file: "__FILE__", line: %d, " + "I am already the trunk server %s:%d, " + "may be the tracker server restart", + __LINE__, pBriefServers->ip_addr, port); } else { - logInfo("file: "__FILE__", line: %d, " \ - "I am the the trunk server %s:%d", __LINE__, \ - g_trunk_server.ip_addr, g_trunk_server.port); + logInfo("file: "__FILE__", line: %d, " + "I am the the trunk server %s:%d", __LINE__, + pBriefServers->ip_addr, port); tracker_fetch_trunk_fid(pTrackerServer); g_if_trunker_self = true; @@ -1331,7 +1360,7 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \ return result; } - if (g_trunk_create_file_advance && \ + if (g_trunk_create_file_advance && g_trunk_create_file_interval > 0) { ScheduleArray scheduleArray; @@ -1353,9 +1382,10 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \ } else { - logInfo("file: "__FILE__", line: %d, " \ - "the trunk server is %s:%d", __LINE__, \ - g_trunk_server.ip_addr, g_trunk_server.port); + logInfo("file: "__FILE__", line: %d, " + "the trunk server is %s:%d", __LINE__, + g_trunk_server.connections[0].ip_addr, + g_trunk_server.connections[0].port); if (g_if_trunker_self) { @@ -1364,8 +1394,8 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \ logWarning("file: "__FILE__", line: %d, " \ "I am the old trunk server, " \ "the new trunk server is %s:%d", \ - __LINE__, g_trunk_server.ip_addr, \ - g_trunk_server.port); + __LINE__, g_trunk_server.connections[0].ip_addr, \ + g_trunk_server.connections[0].port); tracker_report_trunk_fid(pTrackerServer); g_if_trunker_self = false; diff --git a/storage/trunk_mgr/trunk_client.c b/storage/trunk_mgr/trunk_client.c index 1305e55..240ca1b 100644 --- a/storage/trunk_mgr/trunk_client.c +++ b/storage/trunk_mgr/trunk_client.c @@ -93,11 +93,42 @@ static int trunk_client_trunk_do_alloc_space(ConnectionInfo *pTrunkServer, \ return 0; } +static int trunk_client_connect_trunk_server(TrackerServerInfo *trunk_server, + ConnectionInfo **conn, const char *prompt) +{ + int result; + + if (g_trunk_server.count == 0) + { + logError("file: "__FILE__", line: %d, " + "no trunk server", __LINE__); + return EAGAIN; + } + + memcpy(trunk_server, &g_trunk_server, sizeof(TrackerServerInfo)); + if ((*conn=tracker_connect_server(trunk_server, &result)) == NULL) + { + logError("file: "__FILE__", line: %d, " + "%s because connect to trunk " + "server %s:%d fail, errno: %d", __LINE__, + prompt, trunk_server->connections[0].ip_addr, + trunk_server->connections[0].port, result); + return result; + } + + if (g_trunk_server.index != trunk_server->index) + { + g_trunk_server.index = trunk_server->index; + } + + return 0; +} + int trunk_client_trunk_alloc_space(const int file_size, \ FDFSTrunkFullInfo *pTrunkInfo) { int result; - ConnectionInfo trunk_server; + TrackerServerInfo trunk_server; ConnectionInfo *pTrunkServer; if (g_if_trunker_self) @@ -105,22 +136,11 @@ int trunk_client_trunk_alloc_space(const int file_size, \ return trunk_alloc_space(file_size, pTrunkInfo); } - if (*(g_trunk_server.ip_addr) == '\0') - { - logError("file: "__FILE__", line: %d, " \ - "no trunk server", __LINE__); - return EAGAIN; - } - - memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo)); - if ((pTrunkServer=tracker_make_connection(&trunk_server, &result)) == NULL) - { - logError("file: "__FILE__", line: %d, " \ - "can't alloc trunk space because connect to trunk " \ - "server %s:%d fail, errno: %d", __LINE__, \ - trunk_server.ip_addr, trunk_server.port, result); - return result; - } + if ((result=trunk_client_connect_trunk_server(&trunk_server, + &pTrunkServer, "can't alloc trunk space")) != 0) + { + return result; + } result = trunk_client_trunk_do_alloc_space(pTrunkServer, \ file_size, pTrunkInfo); @@ -202,7 +222,7 @@ int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \ const int status) { int result; - ConnectionInfo trunk_server; + TrackerServerInfo trunk_server; ConnectionInfo *pTrunkServer; if (g_if_trunker_self) @@ -210,20 +230,11 @@ int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \ return trunk_alloc_confirm(pTrunkInfo, status); } - if (*(g_trunk_server.ip_addr) == '\0') - { - return EAGAIN; - } - - memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo)); - if ((pTrunkServer=tracker_make_connection(&trunk_server, &result)) == NULL) - { - logError("file: "__FILE__", line: %d, " \ - "trunk alloc confirm fail because connect to trunk " \ - "server %s:%d fail, errno: %d", __LINE__, \ - trunk_server.ip_addr, trunk_server.port, result); - return result; - } + if ((result=trunk_client_connect_trunk_server(&trunk_server, + &pTrunkServer, "trunk alloc confirm fail")) != 0) + { + return result; + } result = trunk_client_trunk_do_alloc_confirm(pTrunkServer, \ pTrunkInfo, status); @@ -235,7 +246,7 @@ int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \ int trunk_client_trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo) { int result; - ConnectionInfo trunk_server; + TrackerServerInfo trunk_server; ConnectionInfo *pTrunkServer; if (g_if_trunker_self) @@ -243,20 +254,11 @@ int trunk_client_trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo) return trunk_free_space(pTrunkInfo, true); } - if (*(g_trunk_server.ip_addr) == '\0') - { - return EAGAIN; - } - - memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo)); - if ((pTrunkServer=tracker_make_connection(&trunk_server, &result)) == NULL) - { - logError("file: "__FILE__", line: %d, " \ - "free trunk space fail because connect to trunk " \ - "server %s:%d fail, errno: %d", __LINE__, \ - trunk_server.ip_addr, trunk_server.port, result); - return result; - } + if ((result=trunk_client_connect_trunk_server(&trunk_server, + &pTrunkServer, "free trunk space fail")) != 0) + { + return result; + } result = trunk_client_trunk_do_free_space(pTrunkServer, pTrunkInfo); tracker_close_connection_ex(pTrunkServer, result != 0); diff --git a/storage/trunk_mgr/trunk_mem.c b/storage/trunk_mgr/trunk_mem.c index f5e1309..ce301a3 100644 --- a/storage/trunk_mgr/trunk_mem.c +++ b/storage/trunk_mgr/trunk_mem.c @@ -54,7 +54,7 @@ int g_current_trunk_file_id = 0; TimeInfo g_trunk_create_file_time_base = {0, 0}; int g_trunk_create_file_interval = 86400; int g_trunk_compress_binlog_min_interval = 0; -ConnectionInfo g_trunk_server = {-1, 0}; +TrackerServerInfo g_trunk_server = {0, 0}; bool g_if_use_trunk_file = false; bool g_if_trunker_self = false; bool g_trunk_create_file_advance = false; @@ -176,9 +176,7 @@ int storage_trunk_init() logDebug("file: "__FILE__", line: %d, " \ "storage trunk init ...", __LINE__); - g_trunk_server.sock = -1; - g_trunk_server.port = g_server_port; - + memset(&g_trunk_server, 0, sizeof(g_trunk_server)); if ((result=init_pthread_lock(&trunk_file_lock)) != 0) { logError("file: "__FILE__", line: %d, " \ diff --git a/storage/trunk_mgr/trunk_mem.h b/storage/trunk_mgr/trunk_mem.h index 7b3a6cc..dc3cac4 100644 --- a/storage/trunk_mgr/trunk_mem.h +++ b/storage/trunk_mgr/trunk_mem.h @@ -37,7 +37,7 @@ extern int g_current_trunk_file_id; //current trunk file id extern TimeInfo g_trunk_create_file_time_base; extern int g_trunk_create_file_interval; extern int g_trunk_compress_binlog_min_interval; -extern ConnectionInfo g_trunk_server; //the trunk server +extern TrackerServerInfo g_trunk_server; //the trunk server extern bool g_if_use_trunk_file; //if use trunk file extern bool g_trunk_create_file_advance; extern bool g_trunk_init_check_occupying; diff --git a/tracker/fdfs_shared_func.c b/tracker/fdfs_shared_func.c index 1424315..9a200a6 100644 --- a/tracker/fdfs_shared_func.c +++ b/tracker/fdfs_shared_func.c @@ -660,3 +660,25 @@ void fdfs_set_multi_ip_index(FDFSMultiIP *multi_ip, const char *target_ip) } } } + +void fdfs_set_server_info(TrackerServerInfo *pServer, + const char *ip_addr, const int port) +{ + pServer->count = 1; + pServer->index = 0; + conn_pool_set_server_info(pServer->connections + 0, ip_addr, port); +} + +void fdfs_set_server_info_ex(TrackerServerInfo *pServer, + const FDFSMultiIP *ip_addrs, const int port) +{ + int i; + + pServer->count = ip_addrs->count; + pServer->index = 0; + for (i=0; icount; i++) + { + conn_pool_set_server_info(pServer->connections + i, + ip_addrs->ips[i], port); + } +} diff --git a/tracker/fdfs_shared_func.h b/tracker/fdfs_shared_func.h index f784ecf..86eece4 100644 --- a/tracker/fdfs_shared_func.h +++ b/tracker/fdfs_shared_func.h @@ -124,6 +124,12 @@ const char *fdfs_get_ipaddr_by_client_ip(const FDFSMultiIP *ip_addrs, void fdfs_set_multi_ip_index(FDFSMultiIP *multi_ip, const char *target_ip); +void fdfs_set_server_info(TrackerServerInfo *pServer, + const char *ip_addr, const int port); + +void fdfs_set_server_info_ex(TrackerServerInfo *pServer, + const FDFSMultiIP *ip_addrs, const int port); + #ifdef __cplusplus } #endif diff --git a/tracker/tracker_proto.c b/tracker/tracker_proto.c index a015cc2..0e646b3 100644 --- a/tracker/tracker_proto.c +++ b/tracker/tracker_proto.c @@ -544,14 +544,14 @@ void tracker_disconnect_server(TrackerServerInfo *pServerInfo) if (pServerInfo->count == 1) { - conn_pool_disconnect_server(pServerInfo->connections); + tracker_close_connection_ex(pServerInfo->connections + 0, true); return; } end = pServerInfo->connections + pServerInfo->count; for (conn=pServerInfo->connections; conn