[EXPORTER] Support handling retry-able errors for OTLP/gRPC (#3219)

Author: chusitoo

CHANGELOG.md

@@ -36,6 +36,9 @@ Increment the:
 * [EXPORTER] Support handling retry-able errors for OTLP/HTTP
   [#3223](https://github.com/open-telemetry/opentelemetry-cpp/pull/3223)
 
+* [EXPORTER] Support handling retry-able errors for OTLP/gRPC
+  [#3219](https://github.com/open-telemetry/opentelemetry-cpp/pull/3219)
+
 New features:
 
 * [SDK] Better control of threads executed by opentelemetry-cpp

ci/do_ci.sh

@@ -347,6 +347,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then
         -DWITH_OTLP_HTTP=ON \
         -DWITH_OTLP_FILE=ON \
         -DWITH_OTLP_GRPC_SSL_MTLS_PREVIEW=ON \
+        -DWITH_OTLP_RETRY_PREVIEW=ON \
         "${SRC_DIR}"
   grpc_cpp_plugin=`which grpc_cpp_plugin`
   proto_make_file="CMakeFiles/opentelemetry_proto.dir/build.make"

exporters/otlp/CMakeLists.txt

@@ -317,7 +317,7 @@ if(BUILD_TESTING)
     add_executable(otlp_grpc_exporter_test test/otlp_grpc_exporter_test.cc)
     target_link_libraries(
       otlp_grpc_exporter_test ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}
-      ${GMOCK_LIB} opentelemetry_exporter_otlp_grpc)
+      ${GMOCK_LIB} opentelemetry_exporter_otlp_grpc gRPC::grpc++)
     gtest_add_tests(
       TARGET otlp_grpc_exporter_test
       TEST_PREFIX exporter.otlp.

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h

@@ -62,6 +62,18 @@ struct OtlpGrpcClientOptions
   // Concurrent requests
   std::size_t max_concurrent_requests;
 #endif
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/src/otlp_grpc_client.cc

@@ -12,6 +12,7 @@
 #include <atomic>
 #include <chrono>
 #include <condition_variable>
+#include <cstdio>
 #include <fstream>
 #include <iterator>
 #include <memory>
@@ -23,6 +24,7 @@
 #include "opentelemetry/common/timestamp.h"
 #include "opentelemetry/ext/http/common/url_parser.h"
 #include "opentelemetry/nostd/function_ref.h"
+#include "opentelemetry/nostd/string_view.h"
 #include "opentelemetry/sdk/common/global_log_handler.h"
 
 OPENTELEMETRY_BEGIN_NAMESPACE
@@ -347,6 +349,49 @@ std::shared_ptr<grpc::Channel> OtlpGrpcClient::MakeChannel(const OtlpGrpcClientO
     grpc_arguments.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);
   }
 
+#ifdef ENABLE_OTLP_RETRY_PREVIEW
+  if (options.retry_policy_max_attempts > 0U &&
+      options.retry_policy_initial_backoff > std::chrono::duration<float>::zero() &&
+      options.retry_policy_max_backoff > std::chrono::duration<float>::zero() &&
+      options.retry_policy_backoff_multiplier > 0.0f)
+  {
+    static const auto kServiceConfigJson = opentelemetry::nostd::string_view{R"(
+    {
+      "methodConfig": [
+        {
+          "name": [{}],
+          "retryPolicy": {
+            "maxAttempts": %0000000000u,
+            "initialBackoff": "%0000000000.1fs",
+            "maxBackoff": "%0000000000.1fs",
+            "backoffMultiplier": %0000000000.1f,
+            "retryableStatusCodes": [
+              "CANCELLED",
+              "DEADLINE_EXCEEDED",
+              "ABORTED",
+              "OUT_OF_RANGE",
+              "DATA_LOSS",
+              "UNAVAILABLE"
+            ]
+          }
+        }
+      ]
+    })"};
+
+    // Allocate string with buffer large enough to hold the formatted json config
+    auto service_config = std::string(kServiceConfigJson.size(), '\0');
+    // Prior to C++17, need to explicitly cast away constness from `data()` buffer
+    std::snprintf(
+        const_cast<decltype(service_config)::value_type *>(service_config.data()),
+        service_config.size(), kServiceConfigJson.data(), options.retry_policy_max_attempts,
+        std::min(std::max(options.retry_policy_initial_backoff.count(), 0.f), 999999999.f),
+        std::min(std::max(options.retry_policy_max_backoff.count(), 0.f), 999999999.f),
+        std::min(std::max(options.retry_policy_backoff_multiplier, 0.f), 999999999.f));
+
+    grpc_arguments.SetServiceConfigJSON(service_config);
+  }
+#endif  // ENABLE_OTLP_RETRY_PREVIEW
+
   if (options.use_ssl_credentials)
   {
     grpc::SslCredentialsOptions ssl_opts;

exporters/otlp/src/otlp_grpc_exporter.cc

@@ -105,7 +105,7 @@ sdk::common::ExportResult OtlpGrpcExporter::Export(
   google::protobuf::ArenaOptions arena_options;
   // It's easy to allocate datas larger than 1024 when we populate basic resource and attributes
   arena_options.initial_block_size = 1024;
-  // When in batch mode, it's easy to export a large number of spans at once, we can alloc a lager
+  // When in batch mode, it's easy to export a large number of spans at once, we can alloc a larger
   // block to reduce memory fragments.
   arena_options.max_block_size = 65536;
   std::unique_ptr<google::protobuf::Arena> arena{new google::protobuf::Arena{arena_options}};

exporters/otlp/src/otlp_grpc_exporter_options.cc

@@ -36,6 +36,11 @@ OtlpGrpcExporterOptions::OtlpGrpcExporterOptions()
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif
+
+  retry_policy_max_attempts       = GetOtlpDefaultTracesRetryMaxAttempts();
+  retry_policy_initial_backoff    = GetOtlpDefaultTracesRetryInitialBackoff();
+  retry_policy_max_backoff        = GetOtlpDefaultTracesRetryMaxBackoff();
+  retry_policy_backoff_multiplier = GetOtlpDefaultTracesRetryBackoffMultiplier();
 }
 
 OtlpGrpcExporterOptions::~OtlpGrpcExporterOptions() {}

exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc

@@ -34,6 +34,11 @@ OtlpGrpcLogRecordExporterOptions::OtlpGrpcLogRecordExporterOptions()
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif
+
+  retry_policy_max_attempts       = GetOtlpDefaultLogsRetryMaxAttempts();
+  retry_policy_initial_backoff    = GetOtlpDefaultLogsRetryInitialBackoff();
+  retry_policy_max_backoff        = GetOtlpDefaultLogsRetryMaxBackoff();
+  retry_policy_backoff_multiplier = GetOtlpDefaultLogsRetryBackoffMultiplier();
 }
 
 OtlpGrpcLogRecordExporterOptions::~OtlpGrpcLogRecordExporterOptions() {}

exporters/otlp/src/otlp_grpc_metric_exporter_options.cc

@@ -35,6 +35,11 @@ OtlpGrpcMetricExporterOptions::OtlpGrpcMetricExporterOptions()
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif
+
+  retry_policy_max_attempts       = GetOtlpDefaultMetricsRetryMaxAttempts();
+  retry_policy_initial_backoff    = GetOtlpDefaultMetricsRetryInitialBackoff();
+  retry_policy_max_backoff        = GetOtlpDefaultMetricsRetryMaxBackoff();
+  retry_policy_backoff_multiplier = GetOtlpDefaultMetricsRetryBackoffMultiplier();
 }
 
 OtlpGrpcMetricExporterOptions::~OtlpGrpcMetricExporterOptions() {}