convex-backend PR 53: Add support for Cloudflare R2 Storage (#34993)

This PR introduces support for Cloudflare R2 by adding a new environment variable, AWS_ENDPOINT_URL, and replacing get_object_attributes with head_object.

From the [S3 documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/get_object_attributes.html), get_object_attributes combines the functionality of HeadObject and ListParts in a single call.
In the original code get_object_attributes was used to get the file size.
Since R2 does not support get_object_attributes, I use head_object instead.

@emmaling27 I would like your help on this.
I wanted to name the env variable S3_ENDPOINT_URL but I read that AWS automatically provides AWS_ENDPOINT_URL to lambda functions.
I can see you use `aws_config::defaults(BehaviorVersion::v2024_03_28())` which is equivalent to `aws_config::from_env()`, which I assumed automatically set defaults based on env variables ? From my tests it doesn't seem to do anything.
So right now I call `.endpoint_url(endpoint_url)` explicitly.
What is the best way to make it optional ? (I don't know much about rust)
Right now it's a mandatory variable ⚠️ which is not what we want.

----

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

Co-authored-by: Spioune <[email protected]>
GitOrigin-RevId: f465784d59ebd1de2bb1de054669a78e5e68b293

Author: 2 people

crates/aws_s3/src/storage.rs

@@ -9,9 +9,9 @@ use async_trait::async_trait;
 use aws_config::retry::RetryConfig;
 use aws_sdk_s3::{
     operation::{
-        get_object_attributes::{
-            GetObjectAttributesError,
-            GetObjectAttributesOutput,
+        head_object::{
+            HeadObjectError,
+            HeadObjectOutput,
         },
         upload_part::builders::UploadPartFluentBuilder,
     },
@@ -26,7 +26,7 @@ use aws_sdk_s3::{
     Client,
 };
 use aws_utils::{
-    must_config_from_env,
+    must_s3_config_from_env,
     s3::S3Client,
 };
 use bytes::Bytes;
@@ -132,7 +132,7 @@ impl<RT: Runtime> S3Storage<RT> {
         key_prefix: String,
         runtime: RT,
     ) -> anyhow::Result<Self> {
-        let config = must_config_from_env()
+        let config = must_s3_config_from_env()
             .context("AWS env variables are required when using S3 storage")?
             .retry_config(RetryConfig::standard())
             .load()
@@ -433,27 +433,22 @@ impl<RT: Runtime> Storage for S3Storage<RT> {
             .as_str()
             .split_once('/')
             .with_context(|| format!("Invalid fully qualified S3 key {:?}", key))?;
-        let result: Result<
-            GetObjectAttributesOutput,
-            aws_sdk_s3::error::SdkError<GetObjectAttributesError>,
-        > = self
+        let result: Result<HeadObjectOutput, aws_sdk_s3::error::SdkError<HeadObjectError>> = self
             .client
-            .get_object_attributes()
+            .head_object()
             .bucket(bucket)
             .key(s3_key)
-            .object_attributes(aws_sdk_s3::types::ObjectAttributes::Checksum)
-            .object_attributes(aws_sdk_s3::types::ObjectAttributes::ObjectSize)
             .send()
             .await;
         match result {
-            Ok(object_attributes) => {
-                let size = object_attributes
-                    .object_size
+            Ok(head_attributes) => {
+                let size = head_attributes
+                    .content_length
                     .context("Object is missing size")? as u64;
                 Ok(Some(ObjectAttributes { size }))
             },
             Err(aws_sdk_s3::error::SdkError::ServiceError(err)) => match err.err() {
-                GetObjectAttributesError::NoSuchKey(_) => Ok(None),
+                HeadObjectError::NotFound(_) => Ok(None),
                 // Other service errors from S3
                 _ => Err(err.into_err().into()),
             },

crates/aws_utils/src/lib.rs

@@ -14,6 +14,9 @@ use aws_types::region::Region;
 
 pub mod s3;
 
+static S3_ENDPOINT_URL: LazyLock<Option<String>> =
+    LazyLock::new(|| env::var("S3_ENDPOINT_URL").ok());
+
 static AWS_ACCESS_KEY_ID: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("AWS_ACCESS_KEY_ID").ok());
 
@@ -41,3 +44,11 @@ pub fn must_config_from_env() -> anyhow::Result<ConfigLoader> {
         .region(region)
         .credentials_provider(credentials))
 }
+
+pub fn must_s3_config_from_env() -> anyhow::Result<ConfigLoader> {
+    let mut config_loader = must_config_from_env()?;
+    if let Some(s3_endpoint_url) = S3_ENDPOINT_URL.clone() {
+        config_loader = config_loader.endpoint_url(s3_endpoint_url);
+    }
+    Ok(config_loader)
+}

crates/aws_utils/src/s3.rs

@@ -8,7 +8,7 @@ use aws_sdk_s3::{
 };
 use futures_async_stream::try_stream;
 
-use crate::must_config_from_env;
+use crate::must_s3_config_from_env;
 
 #[derive(Clone, Debug)]
 pub struct S3Client(pub Client);
@@ -21,7 +21,7 @@ impl S3Client {
             true => RetryConfig::standard(),
             false => RetryConfig::disabled(),
         };
-        let config = must_config_from_env()
+        let config = must_s3_config_from_env()
             .context("AWS env variables are required when using AWS Lambda")?
             .retry_config(retry_config)
             .load()

self-hosted/docker/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       - S3_STORAGE_MODULES_BUCKET=${S3_STORAGE_MODULES_BUCKET:-}
       - S3_STORAGE_FILES_BUCKET=${S3_STORAGE_FILES_BUCKET:-}
       - S3_STORAGE_SEARCH_BUCKET=${S3_STORAGE_SEARCH_BUCKET:-}
+      - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-}
 
     healthcheck:
       test: curl -f http://localhost:3210/version