Merge branch 'main' into gha_release

microsoft · Jul 2, 2024 · c097b4b · c097b4b
2 parents 8dffa4e + 329288c
commit c097b4b
Show file tree

Hide file tree

Showing 25 changed files with 560 additions and 522 deletions.
diff --git a/.daily_canary b/.daily_canary
@@ -3,4 +3,4 @@
  (  V  ) /  .  \  |    +---=---'
 /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
 2024__
-!..!!
+!..!
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -31,62 +31,34 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
         language: ["cpp"]
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      # Initializes the CodeQL tools for scanning.
+      # Done before CodeQL init to let it find the commit successfully
+      - name: Work around git warning
+        run: git config --global --add safe.directory /__w/CCF/CCF
+
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           queries: security-extended
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      #- name: Autobuild
-      #  uses: github/codeql-action/autobuild@v2
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
-
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-      - run: |
-          cd getting_started/setup_vm
-          sudo apt update
-          sudo apt install -y ansible software-properties-common bsdmainutils dnsutils
-          sudo ansible-playbook ccf-dev.yml --extra-vars "platform=virtual" --extra-vars "require_open_enclave=false"
-        name: Install dependencies
 
       - run: |
           set -ex
-          git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF -DCMAKE_C_COMPILER=`which clang-11` -DCMAKE_CXX_COMPILER=`which clang++-11` ..
+          cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF ..
         name: Run CMake
 
       - run: |
           cd build
-          make
+          make -j16
         name: Run Make
 
       - name: Perform CodeQL Analysis

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,7 @@ This is a test release.
 ### Removed
 
 - Removed the existing metrics endpoint and API (`GET /api/metrics`, `get_metrics_v1`). Stats for request execution can instead be gathered by overriding the `EndpointRegistry::handle_event_request_completed()` method.
+- Removed automatic msgpack support from JSON endpoint adapters, and related `include/ccf/serdes.h` file.
 
 ## [5.0.0-dev18]
 

diff --git a/doc/build_apps/api.rst b/doc/build_apps/api.rst
@@ -125,7 +125,7 @@ Historical Queries
 
 .. doxygenclass:: ccf::historical::AbstractStateCache
    :project: CCF
-   :members: set_default_expiry_duration, get_state_at, get_store_at, get_store_range, drop_cached_states
+   :members: set_default_expiry_duration, set_soft_cache_limit, get_state_at, get_store_at, get_store_range, drop_cached_states
 
 .. doxygenstruct:: ccf::historical::State
    :project: CCF

diff --git a/include/ccf/historical_queries_interface.h b/include/ccf/historical_queries_interface.h
@@ -49,6 +49,8 @@ namespace ccf::historical
 
   using ExpiryDuration = std::chrono::seconds;
 
+  using CacheSize = size_t;
+
   /** Stores the progress of historical query requests.
    *
    * A request will generally need to be made multiple times (with the same
@@ -79,6 +81,13 @@ namespace ccf::historical
     virtual void set_default_expiry_duration(
       ExpiryDuration seconds_until_expiry) = 0;
 
+    /** Set the cache limit (in bytes) to evict least recently used requests
+     * from the cache after its size grows beyond this limit. The limit is not
+     * strict. It is estimated based on serialized states' sizes approximation
+     * and is checked once per tick, and so it can overflow for a short time.
+     */
+    virtual void set_soft_cache_limit(CacheSize cache_limit) = 0;
+
     /** EXPERIMENTAL: Set the tracking of deletes on missing keys for historical
      * queries.
      *

diff --git a/include/ccf/http_consts.h b/include/ccf/http_consts.h
@@ -31,7 +31,6 @@ namespace ccf
       namespace contenttype
       {
         static constexpr auto JSON = "application/json";
-        static constexpr auto MSGPACK = "application/msgpack";
         static constexpr auto TEXT = "text/plain";
         static constexpr auto OCTET_STREAM = "application/octet-stream";
         static constexpr auto GRPC = "application/grpc";

diff --git a/include/ccf/json_handler.h b/include/ccf/json_handler.h
@@ -3,26 +3,23 @@
 #pragma once
 
 #include "ccf/endpoint_registry.h"
-#include "ccf/serdes.h"
 
 #include <llhttp/llhttp.h>
 #include <variant>
 
 namespace ccf
 {
   /*
-   * For simple app methods which expect a JSON request, potentially msgpack'd,
-   * these functions do the common decoding of the input and setting of response
-   * fields, to reduce handler complexity and repetition.
+   * For simple app methods which expect a JSON request these functions do the
+   * common decoding of the input and setting of response fields, to reduce
+   * handler complexity and repetition.
    *
    * Rather than:
    * auto foo = [](auto& ctx) {
    *   nlohmann::json params;
-   *   serdes::Pack pack_type;
    *   if (<content-type is JSON>)
    *   {
-   *     params = unpack(ctx.rpc_ctx->get_request_body());
-   *     pack_type = Text;
+   *     params = nlohmann::json::parse(ctx.rpc_ctx->get_request_body());
    *   }
    *   else
    *   {
@@ -35,15 +32,8 @@ namespace ccf
    *     ctx.rpc_ctx->set_response_header(content_type, Text);
    *     ctx.rpc_ctx->set_response_body(error_msg(result));
    *   }
-   *   if (pack_type == Text)
-   *   {
-   *     ctx.rpc_ctx->set_response_header(content_type, JSON);
-   *     ctx.rpc_ctx->set_response_body(pack(result, Text));
-   *   }
-   *   else
-   *   {
-   *     ...
-   *   }
+   *   ctx.rpc_ctx->set_response_header(content_type, JSON);
+   *   ctx.rpc_ctx->set_response_body(result.dump());
    * };
    *
    * it is possible to write the shorter, clearer, return-based lambda:
@@ -65,24 +55,10 @@ namespace ccf
     using JsonAdapterResponse =
       std::variant<ErrorDetails, RedirectDetails, nlohmann::json>;
 
-    char const* pack_to_content_type(serdes::Pack p);
-
-    serdes::Pack detect_json_pack(const std::shared_ptr<ccf::RpcContext>& ctx);
-
-    serdes::Pack get_response_pack(
-      const std::shared_ptr<ccf::RpcContext>& ctx,
-      serdes::Pack request_pack = serdes::Pack::Text);
-
-    nlohmann::json get_params_from_body(
-      const std::shared_ptr<ccf::RpcContext>& ctx, serdes::Pack pack);
-
-    std::pair<serdes::Pack, nlohmann::json> get_json_params(
-      const std::shared_ptr<ccf::RpcContext>& ctx);
+    nlohmann::json get_json_params(const std::shared_ptr<ccf::RpcContext>& ctx);
 
     void set_response(
-      JsonAdapterResponse&& res,
-      std::shared_ptr<ccf::RpcContext>& ctx,
-      serdes::Pack request_packing);
+      JsonAdapterResponse&& res, std::shared_ptr<ccf::RpcContext>& ctx);
   }
 
   jsonhandler::JsonAdapterResponse make_success();

diff --git a/include/ccf/serdes.h b/include/ccf/serdes.h
diff --git a/samples/apps/logging/logging.cpp b/samples/apps/logging/logging.cpp
@@ -460,6 +460,12 @@ namespace loggingapp
         PUBLIC_RECORDS, context, 10000, 20);
       context.get_indexing_strategies().install_strategy(index_per_public_key);
 
+      // According to manual obvervation it's enough to start evicting old
+      // requests on historical perf test, but not too small to get stuck
+      // because of a single request being larget than the cache.
+      constexpr size_t cache_limit = 1024 * 1024 * 10; // MB
+      context.get_historical_state().set_soft_cache_limit(cache_limit);
+
       const ccf::AuthnPolicies auth_policies = {
         ccf::jwt_auth_policy,
         ccf::user_cert_auth_policy,
@@ -1238,8 +1244,6 @@ namespace loggingapp
       auto get_historical = [this](
                               ccf::endpoints::ReadOnlyEndpointContext& ctx,
                               ccf::historical::StatePtr historical_state) {
-        const auto pack = ccf::jsonhandler::detect_json_pack(ctx.rpc_ctx);
-
         // Parse id from query
         const auto parsed_query =
           ccf::http::parse_query(ctx.rpc_ctx->get_request_query());
@@ -1265,7 +1269,7 @@ namespace loggingapp
           LoggingGetHistorical::Out out;
           out.msg = v.value();
           nlohmann::json j = out;
-          ccf::jsonhandler::set_response(std::move(j), ctx.rpc_ctx, pack);
+          ccf::jsonhandler::set_response(std::move(j), ctx.rpc_ctx);
         }
         else
         {
@@ -1295,8 +1299,6 @@ namespace loggingapp
         [this](
           ccf::endpoints::ReadOnlyEndpointContext& ctx,
           ccf::historical::StatePtr historical_state) {
-          const auto pack = ccf::jsonhandler::detect_json_pack(ctx.rpc_ctx);
-
           // Parse id from query
           const auto parsed_query =
             ccf::http::parse_query(ctx.rpc_ctx->get_request_query());
@@ -1323,7 +1325,7 @@ namespace loggingapp
             out.msg = v.value();
             assert(historical_state->receipt);
             out.receipt = ccf::describe_receipt_v1(*historical_state->receipt);
-            ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx, pack);
+            ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx);
           }
           else
           {
@@ -1346,8 +1348,6 @@ namespace loggingapp
         [this](
           ccf::endpoints::ReadOnlyEndpointContext& ctx,
           ccf::historical::StatePtr historical_state) {
-          const auto pack = ccf::jsonhandler::detect_json_pack(ctx.rpc_ctx);
-
           // Parse id from query
           const auto parsed_query =
             ccf::http::parse_query(ctx.rpc_ctx->get_request_query());
@@ -1381,7 +1381,7 @@ namespace loggingapp
             out.receipt = full_receipt;
             out.receipt["leaf_components"].erase("claims_digest");
             // SNIPPET_END: claims_digest_in_receipt
-            ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx, pack);
+            ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx);
           }
           else
           {

diff --git a/src/clients/perf/perf_client.h b/src/clients/perf/perf_client.h
@@ -353,42 +353,6 @@ namespace client
       append_prepared_tx(tx, index);
     }
 
-    void add_prepared_tx(
-      const std::string& method,
-      const nlohmann::json& params,
-      bool expects_commit,
-      const std::optional<size_t>& index,
-      const serdes::Pack& serdes)
-    {
-      auto body = serdes::pack(params, serdes);
-
-      const PreparedTx tx{
-        rpc_connection->gen_request(
-          method,
-          body,
-          serdes == serdes::Pack::Text ?
-            ccf::http::headervalues::contenttype::JSON :
-            ccf::http::headervalues::contenttype::MSGPACK,
-          HTTP_POST,
-          options.bearer_token.size() == 0 ? nullptr :
-                                             options.bearer_token.c_str()),
-        method,
-        expects_commit};
-
-      append_prepared_tx(tx, index);
-    }
-
-    void add_prepared_tx(
-      const std::string& method,
-      const nlohmann::json& params,
-      bool expects_commit,
-      const std::optional<size_t>& index)
-    {
-      const PreparedTx tx{
-        rpc_connection->gen_request(method, params), method, expects_commit};
-      append_prepared_tx(tx, index);
-    }
-
     static size_t total_byte_size(const PreparedTxs& txs)
     {
       return std::accumulate(
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,4 +3,4 @@ @@
      (  V  ) /  .  \  |    +---=---'
     /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
 __
-    !..!!
+    !..!