From 639d388c6da61fdbc85d2c314b0e4a5aa34dbd43 Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Wed, 26 May 2021 15:21:55 +0800 Subject: [PATCH] use libcurl to fetch URL resource --- HISTORY | 3 +- libfastcommon.spec | 5 +- make.sh | 5 ++ src/http_func.c | 127 ++++++++++++++++++++++++++++++++ src/tests/test_normalize_path.c | 31 +++++++- 5 files changed, 167 insertions(+), 4 deletions(-) diff --git a/HISTORY b/HISTORY index 419f31e..404e9c1 100644 --- a/HISTORY +++ b/HISTORY @@ -1,8 +1,9 @@ -Version 1.51 2021-05-25 +Version 1.51 2021-05-26 * fast_mblock.[hc]: support batch alloc and batch free * uniq_skiplist.[hc]: init function add parameter: allocator_use_lock * add function normalize_path_ex and normalize_uri + * use libcurl to fetch URL resource Version 1.50 2021-05-11 * add function is_digital_string diff --git a/libfastcommon.spec b/libfastcommon.spec index eedbf67..39ea314 100644 --- a/libfastcommon.spec +++ b/libfastcommon.spec @@ -14,8 +14,8 @@ Source: http://github.com/happyfish100/libfastcommon/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -#Requires: /sbin/chkconfig -#BuildRequires: perl %{_includedir}/linux/if.h gettext +BuildRequires: libcurl-devel +Requires: libcurl Requires: %__cp %__mv %__chmod %__grep %__mkdir %__install %__id %description @@ -27,6 +27,7 @@ commit version: %{CommitVersion} %package devel Summary: Development header file +Requires: libcurl-devel Requires: %{name}%{?_isa} = %{version}-%{release} %description devel diff --git a/make.sh b/make.sh index 8edca77..616705b 100755 --- a/make.sh +++ b/make.sh @@ -68,6 +68,11 @@ else fi LIBS='-lm -ldl' +if [ -f /usr/include/curl/curl.h ] || [ -f /usr/local/include/curl/curl.h ]; then + CFLAGS="$CFLAGS -DUSE_LIBCURL" + LIBS="$LIBS -lcurl" +fi + uname=`uname` HAVE_VMMETER_H=0 diff --git a/src/http_func.c b/src/http_func.c index 7afb1a2..ec6e6c0 100644 --- a/src/http_func.c +++ b/src/http_func.c @@ -39,6 +39,131 @@ #include "fc_memory.h" #include "http_func.h" +#ifdef USE_LIBCURL +#include + +static bool curl_inited = false; + +typedef struct { + char *buff; + int length; + int alloc_size; + bool dynamic_alloc; +} CurlCallbackArg; + +static size_t curl_write_data(void *ptr, size_t size, + size_t nmemb, void *userdata) +{ + size_t len; + int alloc_size; + char *new_buff; + CurlCallbackArg *cbarg; + + cbarg = (CurlCallbackArg *)userdata; + len = size * nmemb; + if ((cbarg->alloc_size - cbarg->length) < len) { + if (!cbarg->dynamic_alloc) { + return 0; + } + + alloc_size = 2 * cbarg->alloc_size; + while ((alloc_size - cbarg->length) < len) { + alloc_size *= 2; + } + + new_buff = (char *)fc_malloc(alloc_size); + if (new_buff == NULL) { + return 0; + } + + if (cbarg->length > 0) { + memcpy(new_buff, cbarg->buff, cbarg->length); + } + free(cbarg->buff); + + cbarg->buff = new_buff; + cbarg->alloc_size = alloc_size; + } + + memcpy(cbarg->buff + cbarg->length, ptr, len); + cbarg->length += len; + return len; +} + +int get_url_content_ex(const char *url, const int url_len, + const int connect_timeout, const int network_timeout, + int *http_status, char **content, int *content_len, + char *error_info) +{ + CURLcode result; + long response_code; + CURL *curl; + CurlCallbackArg cbarg; + + *error_info = '\0'; + *http_status = 0; + if (!curl_inited) { + if ((result=curl_global_init(CURL_GLOBAL_ALL)) != 0) { + sprintf(error_info, "curl_global_init fail " + "with code: %d", result); + return errno != 0 ? errno : EBUSY; + } + curl_inited = true; + } + + if ((curl=curl_easy_init()) == NULL) { + sprintf(error_info, "curl_easy_init fail"); + return errno != 0 ? errno : EBUSY; + } + + if (*content == NULL) { + cbarg.dynamic_alloc = true; + cbarg.alloc_size = 16 * 1024; + cbarg.buff = (char *)fc_malloc(cbarg.alloc_size); + if (cbarg.buff == NULL) { + return ENOMEM; + } + } else { + cbarg.dynamic_alloc = false; + cbarg.alloc_size = *content_len; + cbarg.buff = *content; + } + cbarg.length = 0; + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, connect_timeout); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, connect_timeout + network_timeout); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cbarg); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + + result = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + curl_easy_cleanup(curl); + + *http_status = response_code; + if (result == CURLE_OK) { + if (cbarg.dynamic_alloc) { + *content = cbarg.buff; + } + *content_len = cbarg.length; + *(*content + *content_len) = '\0'; + return 0; + } else { + sprintf(error_info, "curl_easy_perform fail with code: %d, %s", + result, curl_easy_strerror(result)); + if (cbarg.dynamic_alloc && cbarg.buff != NULL) { + free(cbarg.buff); + } + *content_len = 0; + return EACCES; + } +} + +#else + int get_url_content_ex(const char *url, const int url_len, const int connect_timeout, const int network_timeout, int *http_status, char **content, int *content_len, char *error_info) @@ -270,6 +395,8 @@ int get_url_content_ex(const char *url, const int url_len, return result; } +#endif + int get_url_content(const char *url, const int connect_timeout, \ const int network_timeout, int *http_status, \ char **content, int *content_len, char *error_info) diff --git a/src/tests/test_normalize_path.c b/src/tests/test_normalize_path.c index 40c5fd7..5aa7d2a 100644 --- a/src/tests/test_normalize_path.c +++ b/src/tests/test_normalize_path.c @@ -22,12 +22,15 @@ #include #include "fastcommon/logger.h" #include "fastcommon/shared_func.h" +#include "fastcommon/http_func.h" int main(int argc, char *argv[]) { char full_filename[PATH_MAX]; const char *from; const char *filename; + int filename_len; + int result; if (argc < 3) { fprintf(stderr, "Usage: %s \n", argv[0]); @@ -37,8 +40,34 @@ int main(int argc, char *argv[]) log_init(); from = argv[1]; filename = argv[2]; - normalize_path_ex(from, filename, full_filename, sizeof(full_filename), + filename_len = normalize_path_ex(from, filename, + full_filename, sizeof(full_filename), NORMALIZE_FLAGS_URL_ENABLED_AND_APPEND_PARAMS); printf("%s\n", full_filename); + + if (IS_URL_RESOURCE(full_filename)) { + const int connect_timeout = 2; + const int network_timeout = 30; + char *content; + int content_len; + int http_status; + char error_info[512]; + + content = NULL; + content_len = 0; + result = get_url_content_ex(full_filename, filename_len, + connect_timeout, network_timeout, &http_status, + &content, &content_len, error_info); + if (result == 0) { + printf("http status: %d, content length: %d\n", + http_status, content_len); + } else if (*error_info != '\0') { + fprintf(stderr, "%s\n", error_info); + } else { + fprintf(stderr, "error code: %d\n", result); + } + return result; + } + return 0; }