From 3221ce553099a1697b1182ee3c1603d3d22f55a7 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:07:14 +0100 Subject: [PATCH 01/10] Update workflow files --- .github/workflows/build_and_test.yml | 32 ++++++------- .github/workflows/codeql-analysis.yml | 68 +++++++++++++-------------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9c88f8d..c124217 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1,12 +1,12 @@ name: build_and_test on: - push: + push: branches: - - 'master' - pull_request: + - "master" + pull_request: branches: - - '*' + - "*" jobs: builds: name: "${{ matrix.name }}" @@ -14,21 +14,21 @@ jobs: strategy: matrix: include: - - name: "Ubuntu Jammy clang-14 build" - os: ubuntu-22.04 - cpp_compiler: clang++ + - name: "Ubuntu Jammy clang-14 build" + os: ubuntu-22.04 + cpp_compiler: clang++ - - name: "Ubuntu Jammy gcc-11 build" - os: ubuntu-22.04 - cpp_compiler: g++-11 + - name: "Ubuntu Jammy gcc-11 build" + os: ubuntu-22.04 + cpp_compiler: g++-11 - - name: "Ubuntu Jammy gcc-10 build" - os: ubuntu-22.04 - cpp_compiler: g++-10 + - name: "Ubuntu Jammy gcc-10 build" + os: ubuntu-22.04 + cpp_compiler: g++-10 - - name: "Mac OS clang-14 build" - os: macos-12 - cpp_compiler: clang++ + - name: "Mac OS clang-14 build" + os: macos-12 + cpp_compiler: clang++ steps: - uses: actions/checkout@v3 - name: Install (Ubuntu) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0885300..c69ec25 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,49 +13,49 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [master] pull_request: schedule: - - cron: '24 18 * * 2' + - cron: "24 18 * * 2" jobs: analyze: name: Analyze - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - language: [ 'cpp' ] + language: ["cpp"] steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - - name: Install packages - run: | - sudo apt update - sudo apt install -y --no-install-recommends ninja-build libcurl4-openssl-dev - - - name: Configure - run: > - cmake - -B build - -G Ninja - -DCMAKE_BUILD_TYPE=Release - -DBUILD_SHARED_LIBS=ON - - name: Build - run: cmake --build build - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + - name: Install packages + run: | + sudo apt update + sudo apt install -y --no-install-recommends ninja-build libcurl4-openssl-dev + + - name: Configure + run: > + cmake + -B build + -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DBUILD_SHARED_LIBS=ON + - name: Build + run: cmake --build build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 From e161a05de0fda8b954e8df133752ef59687dfec7 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:12:27 +0100 Subject: [PATCH 02/10] Is g++-12 there? --- .github/workflows/build_and_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c124217..1ccfced 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -18,6 +18,10 @@ jobs: os: ubuntu-22.04 cpp_compiler: clang++ + - name: "Ubuntu Jammy gcc-12 build" + os: ubuntu-22.04 + cpp_compiler: g++-12 + - name: "Ubuntu Jammy gcc-11 build" os: ubuntu-22.04 cpp_compiler: g++-11 From de27505cfcad6cff73ba8e568617dffd9c198753 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:13:54 +0100 Subject: [PATCH 03/10] Clean-up GCC-10 --- .github/workflows/build_and_test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1ccfced..aeea29e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -26,10 +26,6 @@ jobs: os: ubuntu-22.04 cpp_compiler: g++-11 - - name: "Ubuntu Jammy gcc-10 build" - os: ubuntu-22.04 - cpp_compiler: g++-10 - - name: "Mac OS clang-14 build" os: macos-12 cpp_compiler: clang++ From 0d36c086c28a3435a9d4bdce741fd739e272abe4 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:20:32 +0100 Subject: [PATCH 04/10] Introduce CMP0136 new --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6b7af3..b599b11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,11 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") # Adhere the version number to http://semver.org/ project(cpp-ipfs-http-client VERSION 0.6.0 LANGUAGES CXX) +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + # Compile in C++11 mode set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) From 871eef12d7f8bf56434595e133476ece30631945 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:28:02 +0100 Subject: [PATCH 05/10] Increase C++ standard to C++20 --- CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b599b11..5113e9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.11.0) +cmake_minimum_required(VERSION 3.12.0) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") @@ -10,8 +10,8 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() -# Compile in C++11 mode -set(CMAKE_CXX_STANDARD 11) +# Compile in C++20 mode +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) # Add compiler warnings @@ -62,9 +62,8 @@ target_include_directories(${IPFS_API_LIBNAME} # Fetch "JSON for Modern C++" include(FetchContent) -message(STATUS "Fetching nlohmann/JSON") # Retrieve Nlohmann JSON -FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.10.5/json.tar.xz) +FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz) FetchContent_MakeAvailable(json) # libcurl requires additional libs only for static Windows builds From f7d0d70b329e27f1a887d01c3272d37a821954b6 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:28:57 +0100 Subject: [PATCH 06/10] Increase version to 0.7.0 --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5113e9b..43970a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,9 @@ cmake_minimum_required(VERSION 3.12.0) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") # Adhere the version number to http://semver.org/ -project(cpp-ipfs-http-client VERSION 0.6.0 LANGUAGES CXX) +project(cpp-ipfs-http-client + VERSION 0.7.0 + LANGUAGES CXX) # Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") From ff5d88093ead8d85fd9715eb0cdc98f8e149581a Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 18:38:39 +0100 Subject: [PATCH 07/10] Also set CMAKE_C_FLAGS for C code --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43970a1..0f722b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # Add compiler warnings if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^(AppleClang|Clang|GNU)$") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -pedantic -Werror=incompatible-pointer-types") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wpedantic -Wextra -Werror") endif() From 535a1e37b6a6cef3388dabdac3e7420cd9d12944 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Wed, 13 Dec 2023 19:02:23 +0100 Subject: [PATCH 08/10] Improved readme Improve readme with only cmake calls instead of make or ninja. --- README.md | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d045977..6f5e9f0 100644 --- a/README.md +++ b/README.md @@ -52,19 +52,22 @@ When building documention, you also need: - [Doxygen](https://www.doxygen.nl/download.html) (>= v1.9.0) -## Install +## Build & Install ```sh git clone https://github.com/vasild/cpp-ipfs-http-client.git cd cpp-ipfs-http-client -mkdir build -cd build -cmake .. -make -j 8 -sudo make install +# Configure build +cmake -B build +# Build +cmake --build build -j 8 +# Install +sudo cmake --install build ``` -*Hint:* Only build the library without tests, use: `cmake -DBUILD_TESTING=OFF ..` +*Hint #1:* You can also build using **Ninja** (iso Make), use the following as configure: `cmake -GNinja -B build`, then use: `cmake --build build` which will use Ninja, _no need for `-j` anymore_. +*Hint #2:* Build a specific target (eg. ipfs-http-client), use: `cmake --build build --target ipfs-http-client -j 8` +*Hint #3:* You could also build the library without tests, use the option: `cmake -DBUILD_TESTING=OFF -B build` See the [documentation for details](https://vasild.github.io/cpp-ipfs-http-client). @@ -73,11 +76,10 @@ See the [documentation for details](https://vasild.github.io/cpp-ipfs-http-clien Only build & run the test cases, *without* code coverage: ```sh -mkdir build && cd build -cmake .. -make -j 8 -# Run our test-cases -make our_tests +# Prepare +cmake -B build +# Build & run our tests +cmake --build build --target our_tests -j 8 ``` ## Build & run Test cases + Code Coverage @@ -85,14 +87,12 @@ make our_tests Test cases are build by default, but if you want to build with coverage: ```sh -mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON -B build +# Build & run tests + HTML report +cmake --build build --target ctest_coverage_html -j 8 -cmake -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON.. -# Run tests & Build the HTML report -make ctest_coverage_html -j 8 - -# Or run tests & create a Cobertura XML file -make ctest_coverage_xml -j 8 +# Or build & run tests + Cobertura XML file +cmake --build build --target ctest_coverage_xml -j 8 ``` ## Build Doxygen @@ -100,9 +100,8 @@ make ctest_coverage_xml -j 8 Build Doxygen files locally. From the root directory of this project: ```sh -mkdir build && cd build -cmake -DDOC=ON .. -make doc +cmake -DDOC=ON -B build +cmake --build build --target doc ``` ## Usage From f300abb7fba211b25f9e7717db2c862bfef81e07 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Thu, 14 Dec 2023 19:05:12 +0100 Subject: [PATCH 09/10] Fix deprecated calls to curl (move to curl MIME API) (#69) * First try to migrate to the new mime api * Fix mime implementation * remove comment * Fix clang format * Clean-up --- include/ipfs/http/transport-curl.h | 3 + src/http/transport-curl.cc | 105 +++++++++++++++-------------- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/include/ipfs/http/transport-curl.h b/include/ipfs/http/transport-curl.h index 75fdbc9..06e4aba 100644 --- a/include/ipfs/http/transport-curl.h +++ b/include/ipfs/http/transport-curl.h @@ -146,6 +146,9 @@ class TransportCurl : public Transport { /** cURL multi handle. */ CURLM* multi_handle_; + /** cURL mime structure for post requests */ + curl_mime* multipart_; + /** Flag for enabling CURL verbose mode, useful for debugging */ bool curl_verbose_; diff --git a/src/http/transport-curl.cc b/src/http/transport-curl.cc index 33960d6..4a74223 100644 --- a/src/http/transport-curl.cc +++ b/src/http/transport-curl.cc @@ -160,9 +160,9 @@ std::unique_ptr TransportCurl::Clone() const { TransportCurl::~TransportCurl() { if (multi_handle_ && curl_) { - curl_multi_remove_handle(multi_handle_, curl_); - curl_multi_cleanup(multi_handle_); - curl_easy_cleanup(curl_); + curl_multi_remove_handle(multi_handle_, curl_); // Remove easy handle + curl_easy_cleanup(curl_); // Clean up all easy handles + curl_multi_cleanup(multi_handle_); // Finally remove the multi handle curl_global_cleanup(); } } @@ -171,55 +171,46 @@ void TransportCurl::Fetch(const std::string& url, const std::vector& files, std::iostream* response) { /* https://curl.se/libcurl/c/CURLOPT_POST.html */ - curl_easy_setopt(curl_, CURLOPT_POST, 1); - - curl_httppost* form_parts = NULL; - curl_httppost* form_parts_end = NULL; - - for (size_t i = 0; i < files.size(); ++i) { - const FileUpload& file = files[i]; - const std::string name("file" + std::to_string(i)); - static const char* content_type = "application/octet-stream"; - - switch (file.type) { - case FileUpload::Type::kFileContents: - /* https://curl.se/libcurl/c/curl_formadd.html */ - curl_formadd(&form_parts, &form_parts_end, - /* name="..."; */ - CURLFORM_COPYNAME, name.c_str(), - /* filename="..."; */ - CURLFORM_BUFFER, file.path.c_str(), - /* File contents. */ - CURLFORM_BUFFERPTR, file.data.c_str(), - /* File contents length. */ - CURLFORM_BUFFERLENGTH, file.data.length(), - /* Content-Type: */ - CURLFORM_CONTENTTYPE, content_type, CURLFORM_END); - break; - case FileUpload::Type::kFileName: - /* https://curl.se/libcurl/c/curl_formadd.html */ - curl_formadd(&form_parts, &form_parts_end, - /* name="..."; */ - CURLFORM_COPYNAME, name.c_str(), - /* filename="..."; */ - CURLFORM_FILENAME, file.path.c_str(), - /* Read the data from this file. */ - CURLFORM_FILE, file.data.c_str(), - /* Content-Type: */ - CURLFORM_CONTENTTYPE, content_type, CURLFORM_END); - break; + curl_easy_setopt(curl_, CURLOPT_POST, 1L); + + multipart_ = curl_mime_init(curl_); + + if (multipart_) { + for (size_t i = 0; i < files.size(); ++i) { + const FileUpload& file = files[i]; + const std::string name("file" + std::to_string(i)); + static const char* content_type = "application/octet-stream"; + curl_mimepart* part; + + switch (file.type) { + case FileUpload::Type::kFileContents: + /* Add a part. + * https://curl.se/libcurl/c/curl_mime_addpart.html */ + part = curl_mime_addpart(multipart_); + curl_mime_name(part, name.c_str()); + /* Memory source: */ + curl_mime_data(part, file.data.c_str(), file.data.length()); + curl_mime_filename(part, file.path.c_str()); + curl_mime_type(part, content_type); + break; + case FileUpload::Type::kFileName: + /* Add a part. + * https://curl.se/libcurl/c/curl_mime_addpart.html */ + part = curl_mime_addpart(multipart_); + curl_mime_name(part, name.c_str()); + /* File source: */ + curl_mime_filedata(part, file.data.c_str()); + // Override filename (instead of using the remote file name) + curl_mime_filename(part, file.path.c_str()); + curl_mime_type(part, content_type); + break; + } } - } - /* Auto free the resources occupied by `form_parts`. */ - std::unique_ptr form_parts_deleter( - form_parts, [](curl_httppost* d) { - /* https://curl.se/libcurl/c/curl_formfree.html */ - curl_formfree(d); - }); - - /* https://curl.se/libcurl/c/CURLOPT_HTTPPOST.html */ - curl_easy_setopt(curl_, CURLOPT_HTTPPOST, form_parts); + /* Set the form info + * https://curl.se/libcurl/c/CURLOPT_MIMEPOST.html */ + curl_easy_setopt(curl_, CURLOPT_MIMEPOST, multipart_); + } curl_slist* headers = NULL; /* https://curl.se/libcurl/c/curl_slist_append.html */ @@ -346,6 +337,20 @@ void TransportCurl::Perform(const std::string& url, std::iostream* response) { * https://curl.se/libcurl/c/curl_multi_remove_handle.html */ curl_multi_remove_handle(multi_handle_, curl_); + /* + * Note: We re-use easy curl handle and multiple handle, so we don't clean it + * up here. + */ + + /* Reset the easy to default settings, so we can safely reuse the handle */ + curl_easy_reset(curl_); + + /* If post multi part mime structure was used, free it. + https://curl.se/libcurl/c/curl_mime_free.html */ + if (multipart_) { + curl_mime_free(multipart_); + } + /* If there were errors, throw them now (if atomic bool is still true) */ if (keep_perform_running_) { if (!generic_error.empty()) { From 7a906c44ba1d9623c0596f713cd63aaabcdf91c1 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Fri, 15 Dec 2023 01:28:12 +0100 Subject: [PATCH 10/10] Update to doxygen 1.9.8 (#70) --- .github/workflows/build_and_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index aeea29e..32253a2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -106,11 +106,11 @@ jobs: run: | sudo apt update sudo apt install -y --no-install-recommends libcurl4-openssl-dev - - name: Install latest Doxygen (v1.9.7) + - name: Install latest Doxygen (v1.9.8) run: | - wget https://www.doxygen.nl/files/doxygen-1.9.7.linux.bin.tar.gz - tar -xf doxygen-1.9.7.linux.bin.tar.gz - cd doxygen-1.9.7 + wget https://www.doxygen.nl/files/doxygen-1.9.8.linux.bin.tar.gz + tar -xf doxygen-1.9.8.linux.bin.tar.gz + cd doxygen-1.9.8 sudo make install - name: Configure run: >