Deploy webtransport to k8s (#88)

* save

* save

* cargo fmt

* save

* add webtransport enabled flag

* save

* revert cert changes

* Turn args into env vars and add converter from pem to der at startup

* Deploying webtransport to k8s - wip

* save

* Use two loadbalancers

* cargo fmt

* include wildcard in cert

* push ingress

* Add port to webtransport

* increase replicas

* 1 pod per node for webtransport

* add better error handling

* added scripts to automate cutting images

* remove tcp load balancer

* add traces back

* use experimental version without final drop

* save

* bump yew-webtransport

---------

Co-authored-by: Dario Lencina <[email protected]>

Author: griffobeid

Dockerfile.actix

@@ -12,10 +12,11 @@ COPY . /app
 WORKDIR /app/actix-api
 RUN cargo build --release
 
-FROM debian:bullseye-slim as production
+FROM --platform=linux/amd64 debian:bullseye-slim as production
 
 COPY --from=build /usr/bin/dbmate /usr/bin/dbmate
-COPY --from=build /app/actix-api/target/release/actix-api /usr/bin/actix-api
+COPY --from=build /app/actix-api/target/release/websocket_server /usr/bin/websocket_server
+COPY --from=build /app/actix-api/target/release/webtransport_server /usr/bin/webtransport_server
 COPY --from=build /app/actix-api/startup.sh /usr/bin/startup.sh
 COPY --from=build /app/dbmate /app/dbmate
 

Dockerfile.yew

@@ -12,6 +12,8 @@ FROM --platform=linux/amd64  development as build
 ENV ENABLE_OAUTH=false
 ENV LOGIN_URL=""
 ENV ACTIX_UI_BACKEND_URL="wss://api.rustlemania.com"
+ENV WEBTRANSPORT_HOST="https://transport.rustlemania.com:4433"
+ENV WEBTRANSPORT_ENABLED="false"
 WORKDIR /app
 COPY . .
 WORKDIR /app/yew-ui

actix-api/Cargo.lock

@@ -19,6 +19,7 @@ path = "src/bin/websocket_server.rs"
 actix = "0.13.0"
 actix-cors = "0.6.1"
 actix-http = "3.2.2"
+actix-rt = "2.8.0"
 actix-web = "4.1.0"
 actix-web-actors = "4.1.0"
 anyhow = "1.0.60"
@@ -41,10 +42,10 @@ rayon = "1.7.0"
 reqwest = { version = "0.11.11", features = ["json"]}
 rustls = { version = "0.21.2", features = ["dangerous_configuration"] }
 rustls-native-certs = "0.6.3"
-sec-http3 = "0.1.1"
+rustls-pemfile = "1.0.3"
+sec-http3 = { git= "https://github.com/security-union/sec-http3.git", rev = "4c505a86b98da2aaa21cd17f37908e4ce9b64f3a" }
 serde = "1.0.140"
 serde_json = "1.0.82"
-structopt = "0.3.26"
 tokio = { version = "1.28.2", features = ["full"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["fmt", "ansi", "env-filter", "time", "tracing-log"] }

actix-api/Cargo.toml

@@ -1,20 +1,54 @@
-use structopt::StructOpt;
+use std::net::ToSocketAddrs;
+
+use actix_web::{web, App, HttpResponse, HttpServer, Responder};
 use tracing::{error, info};
 
-use sec_api::webtransport;
+use sec_api::webtransport::{self, Certs};
+
+async fn health_responder() -> impl Responder {
+    HttpResponse::Ok().body("Ok")
+}
 
-#[tokio::main]
+#[actix_rt::main]
 async fn main() {
     // Turned this off because it's too verbose
-    // tracing_subscriber::fmt()
-    //     .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-    //     .with_span_events(tracing_subscriber::fmt::format::FmtSpan::FULL)
-    //     .with_writer(std::io::stderr)
-    //     .init();
+    tracing_subscriber::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .with_span_events(tracing_subscriber::fmt::format::FmtSpan::FULL)
+        .with_writer(std::io::stderr)
+        .init();
+
+    let opt = webtransport::WebTransportOpt {
+        listen: std::env::var("LISTEN_URL")
+            .expect("expected LISTEN_URL to be set")
+            .to_socket_addrs()
+            .expect("expected LISTEN_URL to be a valid socket address")
+            .next()
+            .expect("expected LISTEN_URL to be a valid socket address"),
+        certs: Certs {
+            key: std::env::var("KEY_PATH")
+                .expect("expected KEY_PATH to be set")
+                .into(),
+            cert: std::env::var("CERT_PATH")
+                .expect("expected CERT_PATH to be set")
+                .into(),
+        },
+    };
+
+    let listen = opt.listen.clone();
+    actix_rt::spawn(async move {
+        info!("Starting http server: {:?}", listen);
+        let server =
+            HttpServer::new(|| App::new().route("/healthz", web::get().to(health_responder)))
+                .bind(&listen)
+                .unwrap();
+        if let Err(e) = server.run().await {
+            error!("http server error: {}", e);
+        }
+    });
 
-    let opt = webtransport::WebTransportOpt::from_args();
-    match webtransport::start(opt).await {
-        Ok(_) => info!("webtransport server stopped"),
-        Err(e) => error!("webtransport server error: {}", e),
-    }
+    actix_rt::spawn(async move {
+        webtransport::start(opt).await.unwrap();
+    })
+    .await;
 }