s3 example - thumbnail creator (awslabs#613)

peterborkuti · peterborkuti · commit b6d0ed647848 · 2023-03-19T20:18:59.000+01:00
diff --git a/examples/basic-s3-thumbnail/Cargo.toml b/examples/basic-s3-thumbnail/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "basic-s3-thumbnail"
+version = "0.1.0"
+edition = "2021"
+
+# Starting in Rust 1.62 you can use `cargo add` to add dependencies 
+# to your project.
+#
+# If you're using an older Rust version,
+# download cargo-edit(https://github.com/killercup/cargo-edit#installation) 
+# to install the `add` subcommand.
+#
+# Running `cargo add DEPENDENCY_NAME` will
+# add the latest version of a dependency to the list,
+# and it will keep the alphabetic ordering for you.
+
+[dependencies]
+aws_lambda_events = "0.7.2"
+lambda_runtime = { path = "../../lambda-runtime" }
+serde = "1.0.136"
+tokio = { version = "1", features = ["macros"] }
+tracing = { version = "0.1" }
+tracing-subscriber = { version = "0.3", default-features = false, features = ["ansi", "fmt"] }
+aws-config = "0.54.1"
+aws-sdk-s3 = "0.24.0"
+thumbnailer = "0.4.0"
+mime = "0.3.16"
+async-trait = "0.1.66"
+
+[dev-dependencies]
+mockall = "0.11.3"
+tokio-test = "0.4.2"
diff --git a/examples/basic-s3-thumbnail/README.md b/examples/basic-s3-thumbnail/README.md
@@ -0,0 +1,16 @@
+# AWS Lambda Function that uses S3
+
+This example processes S3 events. If the event is a CREATE event,
+it downloads the created file, generates a thumbnail from it
+(it assumes that the file is an image) and uploads it to S3 into a bucket named
+[original-bucket-name]-thumbs.
+
+## Build & Deploy
+
+1. Install [cargo-lambda](https://github.com/cargo-lambda/cargo-lambda#installation)
+2. Build the function with `cargo lambda build --release` 
+3. Deploy the function to AWS Lambda with `cargo lambda deploy --iam-role YOUR_ROLE`
+
+## Build for ARM 64
+
+Build the function with `cargo lambda build --release --arm64` 
diff --git a/examples/basic-s3-thumbnail/src/main.rs b/examples/basic-s3-thumbnail/src/main.rs
@@ -0,0 +1,234 @@
+use aws_config::meta::region::RegionProviderChain;
+use aws_lambda_events::{event::s3::S3Event, s3::S3EventRecord};
+use aws_sdk_s3::Client as S3Client;
+use lambda_runtime::{run, service_fn, Error, LambdaEvent};
+use s3client::{GetFile, GetThumbnail, PutFile};
+
+mod s3client;
+
+/**
+This lambda handler
+    * listen to file creation events
+    * downloads the created file
+    * creates a thumbnail from it
+    * uploads the thumbnail to bucket "[original bucket name]-thumbs".
+
+Make sure that
+    * the created png file has no strange characters in the name
+    * there is another bucket with "-thumbs" suffix in the name
+    * this lambda only gets event from png file creation
+    * this lambda has permission to put file into the "-thumbs" bucket
+*/
+pub(crate) async fn function_handler<T: PutFile + GetFile + GetThumbnail>(
+    event: LambdaEvent<S3Event>,
+    client: &T,
+) -> Result<String, String> {
+    let result = Ok("".to_string());
+    let records = event.payload.records;
+    for record in records.iter() {
+        let (bucket, key) = get_file_props(record);
+        if bucket.is_empty() || key.is_empty() {
+            // The event is not a create event or bucket/object key is missing
+            println!("record skipped");
+            continue;
+        }
+
+        let reader = client.get_file(&bucket, &key).await;
+
+        if reader.is_none() {
+            continue;
+        }
+
+        let thumbnail = client.get_thumbnail(reader.unwrap());
+
+        let mut thumbs_bucket = bucket.to_owned();
+        thumbs_bucket.push_str("-thumbs");
+
+        // It uplaods the thumbnail into a bucket name suffixed with "-thumbs"
+        // So it needs file creation permission into that bucket
+
+        return client.put_file(&thumbs_bucket, &key, thumbnail).await;
+    }
+
+    return result;
+}
+
+fn get_file_props(record: &S3EventRecord) -> (String, String) {
+    let empty_response = ("".to_string(), "".to_string());
+
+    if record.event_name.is_none() {
+        return empty_response;
+    }
+    if !record.event_name.as_ref().unwrap().starts_with("ObjectCreated") {
+        return empty_response;
+    }
+
+    if record.s3.bucket.name.is_none() || record.s3.object.key.is_none() {
+        return empty_response;
+    }
+
+    let bucket_name = record.s3.bucket.name.to_owned().unwrap();
+    let object_key = record.s3.object.key.to_owned().unwrap();
+
+    if bucket_name.is_empty() || object_key.is_empty() {
+        println!("Bucket name or object_key is empty");
+        return empty_response;
+    }
+
+    println!("Bucket: {}, Object key: {}", bucket_name, object_key);
+
+    return (bucket_name, object_key);
+}
+
+async fn get_client() -> S3Client {
+    let region_provider = RegionProviderChain::default_provider().or_else("us-east-2");
+    let config = aws_config::from_env().region(region_provider).load().await;
+    let client = S3Client::new(&config);
+
+    println!("client region {}", client.conf().region().unwrap().to_string());
+
+    return client;
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    set_tracing();
+    set_handler().await
+}
+
+async fn set_handler() -> Result<(), Error> {
+    let client = get_client().await;
+    let client_ref = &client;
+
+    let func = service_fn(move |event| async move { function_handler(event, client_ref).await });
+
+    run(func).await?;
+
+    Ok(())
+}
+
+fn set_tracing() {
+    // required to enable CloudWatch error logging by the runtime
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::INFO)
+        // disable printing the name of the module in every log line.
+        .with_target(false)
+        // this needs to be set to false, otherwise ANSI color codes will
+        // show up in a confusing manner in CloudWatch logs.
+        .with_ansi(false)
+        // disabling time is handy because CloudWatch will add the ingestion time.
+        .without_time()
+        .init();
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::io::Cursor;
+
+    use super::*;
+    use async_trait::async_trait;
+    use aws_lambda_events::chrono::DateTime;
+    use aws_lambda_events::s3::S3Bucket;
+    use aws_lambda_events::s3::S3Entity;
+    use aws_lambda_events::s3::S3Object;
+    use aws_lambda_events::s3::S3RequestParameters;
+    use aws_lambda_events::s3::S3UserIdentity;
+    use aws_sdk_s3::types::ByteStream;
+    use lambda_runtime::{Context, LambdaEvent};
+    use mockall::mock;
+    use mockall::predicate::eq;
+    use s3client::GetFile;
+    use s3client::PutFile;
+
+    #[tokio::test]
+    async fn response_is_good() {
+        let mut context = Context::default();
+        context.request_id = "test-request-id".to_string();
+
+        let bucket = "test-bucket";
+        let key = "test-key";
+
+        mock! {
+            FakeS3Client {}
+
+            #[async_trait]
+            impl GetFile for FakeS3Client {
+                pub async fn get_file(&self, bucket: &str, key: &str) -> Option<Cursor<Vec<u8>>>;
+            }
+            #[async_trait]
+            impl PutFile for FakeS3Client {
+                pub async fn put_file(&self, bucket: &str, key: &str, bytes: ByteStream) -> Result<String, String>;
+            }
+
+            impl GetThumbnail for FakeS3Client {
+                fn get_thumbnail(&self, reader: Cursor<Vec<u8>>) -> ByteStream;
+            }
+        }
+
+        let mut mock = MockFakeS3Client::new();
+
+        mock.expect_get_file()
+            .withf(|b: &str, k: &str| b.eq(bucket) && k.eq(key))
+            .returning(|_1, _2| Some(Cursor::new(b"IMAGE".to_vec())));
+
+        mock.expect_get_thumbnail()
+            .with(eq(Cursor::new(b"IMAGE".to_vec())))
+            .returning(|_| ByteStream::from_static(b"THUMBNAIL"));
+
+        mock.expect_put_file()
+            .withf(|bu: &str, ke: &str, _by| bu.eq("test-bucket-thumbs") && ke.eq(key))
+            .returning(|_1, _2, _3| Ok("Done".to_string()));
+
+        let payload = get_s3_event("ObjectCreated", bucket, key);
+        let event = LambdaEvent { payload, context };
+
+        let result = function_handler(event, &mock).await.unwrap();
+
+        assert_eq!("Done", result);
+    }
+
+    fn get_s3_event(event_name: &str, bucket_name: &str, object_key: &str) -> S3Event {
+        return S3Event {
+            records: (vec![get_s3_event_record(event_name, bucket_name, object_key)]),
+        };
+    }
+
+    fn get_s3_event_record(event_name: &str, bucket_name: &str, object_key: &str) -> S3EventRecord {
+        let s3_entity = S3Entity {
+            schema_version: (Some(String::default())),
+            configuration_id: (Some(String::default())),
+            bucket: (S3Bucket {
+                name: (Some(bucket_name.to_string())),
+                owner_identity: (S3UserIdentity {
+                    principal_id: (Some(String::default())),
+                }),
+                arn: (Some(String::default())),
+            }),
+            object: (S3Object {
+                key: (Some(object_key.to_string())),
+                size: (Some(1)),
+                url_decoded_key: (Some(String::default())),
+                version_id: (Some(String::default())),
+                e_tag: (Some(String::default())),
+                sequencer: (Some(String::default())),
+            }),
+        };
+
+        return S3EventRecord {
+            event_version: (Some(String::default())),
+            event_source: (Some(String::default())),
+            aws_region: (Some(String::default())),
+            event_time: (DateTime::default()),
+            event_name: (Some(event_name.to_string())),
+            principal_id: (S3UserIdentity {
+                principal_id: (Some("X".to_string())),
+            }),
+            request_parameters: (S3RequestParameters {
+                source_ip_address: (Some(String::default())),
+            }),
+            response_elements: (HashMap::new()),
+            s3: (s3_entity),
+        };
+    }
+}
diff --git a/examples/basic-s3-thumbnail/src/s3client.rs b/examples/basic-s3-thumbnail/src/s3client.rs
@@ -0,0 +1,78 @@
+use std::io::Cursor;
+
+use async_trait::async_trait;
+use aws_sdk_s3::{types::ByteStream, Client as S3Client};
+use thumbnailer::{create_thumbnails, ThumbnailSize};
+
+#[async_trait]
+pub trait GetFile {
+    async fn get_file(&self, bucket: &str, key: &str) -> Option<Cursor<Vec<u8>>>;
+}
+
+#[async_trait]
+pub trait PutFile {
+    async fn put_file(&self, bucket: &str, key: &str, bytes: ByteStream) -> Result<String, String>;
+}
+
+pub trait GetThumbnail {
+    fn get_thumbnail(&self, reader: Cursor<Vec<u8>>) -> ByteStream;
+}
+
+impl GetThumbnail for S3Client {
+    fn get_thumbnail(&self, reader: Cursor<Vec<u8>>) -> ByteStream {
+        let mut thumbnails = create_thumbnails(reader, mime::IMAGE_PNG, [ThumbnailSize::Small]).unwrap();
+
+        let thumbnail = thumbnails.pop().unwrap();
+        let mut buf = Cursor::new(Vec::new());
+        thumbnail.write_png(&mut buf).unwrap();
+
+        return ByteStream::from(buf.into_inner());
+    }
+}
+
+#[async_trait]
+impl GetFile for S3Client {
+    async fn get_file(&self, bucket: &str, key: &str) -> Option<Cursor<Vec<u8>>> {
+        println!("get file bucket {}, key {}", bucket, key);
+
+        let output = self.get_object().bucket(bucket).key(key).send().await;
+
+        let mut reader = None;
+
+        if output.as_ref().ok().is_some() {
+            let bytes = output.ok().unwrap().body.collect().await.unwrap().to_vec();
+            println!("Object is downloaded, size is {}", bytes.len());
+            reader = Some(Cursor::new(bytes));
+        } else if output.as_ref().err().is_some() {
+            let err = output.err().unwrap();
+            let service_err = err.into_service_error();
+            let meta = service_err.meta();
+            println!("Error from aws when downloding: {}", meta.to_string());
+        } else {
+            println!("Unknown error when downloading");
+        }
+
+        return reader;
+    }
+}
+
+#[async_trait]
+impl PutFile for S3Client {
+    async fn put_file(&self, bucket: &str, key: &str, bytes: ByteStream) -> Result<String, String> {
+        println!("put file bucket {}, key {}", bucket, key);
+        let result = self.put_object().bucket(bucket).key(key).body(bytes).send().await;
+
+        if result.as_ref().is_ok() {
+            return Ok(format!("Uploaded a file with key {} into {}", key, bucket));
+        }
+
+        return Err(result
+            .err()
+            .unwrap()
+            .into_service_error()
+            .meta()
+            .message()
+            .unwrap()
+            .to_string());
+    }
+}
