From 4fb5c7a6f51fcfb0616782ef85ffd74902c40911 Mon Sep 17 00:00:00 2001 From: itowlson Date: Tue, 23 Jul 2024 11:18:26 +1200 Subject: [PATCH] Validate against target environments during build Signed-off-by: itowlson --- Cargo.lock | 145 ++++++- Cargo.toml | 1 + crates/build/Cargo.toml | 1 + crates/build/src/lib.rs | 105 ++++- crates/build/src/manifest.rs | 125 +++++- crates/compose/src/lib.rs | 111 +++-- crates/environments/Cargo.toml | 38 ++ crates/environments/src/environment.rs | 196 +++++++++ .../src/environment/definition.rs | 125 ++++++ .../src/environment/env_loader.rs | 314 ++++++++++++++ .../environments/src/environment/lockfile.rs | 124 ++++++ crates/environments/src/lib.rs | 227 ++++++++++ crates/environments/src/loader.rs | 246 +++++++++++ crates/loader/src/lib.rs | 2 + crates/loader/src/local.rs | 402 ++++++++++-------- crates/manifest/src/compat.rs | 1 + crates/manifest/src/schema/v2.rs | 29 +- examples/spin-timer/Cargo.lock | 39 ++ src/commands/build.rs | 26 +- src/commands/registry.rs | 4 +- src/commands/up.rs | 4 +- src/commands/up/app_source.rs | 4 +- 22 files changed, 2031 insertions(+), 238 deletions(-) create mode 100644 crates/environments/Cargo.toml create mode 100644 crates/environments/src/environment.rs create mode 100644 crates/environments/src/environment/definition.rs create mode 100644 crates/environments/src/environment/env_loader.rs create mode 100644 crates/environments/src/environment/lockfile.rs create mode 100644 crates/environments/src/lib.rs create mode 100644 crates/environments/src/loader.rs diff --git a/Cargo.lock b/Cargo.lock index 29a2d8ee74..488c48e021 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -5085,6 +5085,7 @@ checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" dependencies = [ "cfg-if", "miette-derive 7.2.0", + "serde", "thiserror 1.0.69", "unicode-width 0.1.14", ] @@ -5562,6 +5563,30 @@ dependencies = [ "unicase", ] +[[package]] +name = "oci-distribution" +version = "0.11.0" +source = "git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba#7e4ce9be9bcd22e78a28f06204931f10c44402ba" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "http 1.1.0", + "http-auth", + "jwt 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.5.0", + "olpc-cjson", + "regex", + "reqwest 0.12.9", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", + "unicase", +] + [[package]] name = "oci-spec" version = "0.7.1" @@ -7880,6 +7905,7 @@ dependencies = [ "anyhow", "serde", "spin-common", + "spin-environments", "spin-manifest", "subprocess", "terminal", @@ -7931,6 +7957,7 @@ dependencies = [ "spin-build", "spin-common", "spin-doctor", + "spin-environments", "spin-factor-outbound-networking", "spin-http", "spin-loader", @@ -8055,6 +8082,40 @@ dependencies = [ "ui-testing", ] +[[package]] +name = "spin-environments" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "futures", + "futures-util", + "id-arena", + "indexmap 2.7.1", + "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba)", + "semver", + "serde", + "serde_json", + "spin-common", + "spin-componentize", + "spin-compose", + "spin-loader", + "spin-manifest", + "spin-serde", + "tokio", + "toml", + "tracing", + "wac-parser", + "wac-resolver", + "wac-types", + "wasm-pkg-client", + "wasmparser 0.229.0", + "wit-component 0.229.0", + "wit-parser 0.229.0", +] + [[package]] name = "spin-expressions" version = "3.3.0-pre0" @@ -8505,7 +8566,7 @@ dependencies = [ "docker_credential", "futures-util", "itertools 0.14.0", - "oci-distribution", + "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7b291a39f74d1a3c9499d934a56cae6580fc8e37)", "reqwest 0.12.9", "serde", "serde_json", @@ -10061,6 +10122,49 @@ dependencies = [ "wasmparser 0.202.0", ] +[[package]] +name = "wac-parser" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec0c4f63641fa095b4a551263fe35a15c72c9680b650b8f08f70db0fdbd19" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.7.1", + "log", + "logos", + "miette 7.2.0", + "semver", + "serde", + "thiserror 1.0.69", + "wac-graph", + "wasm-encoder 0.202.0", + "wasm-metadata 0.202.0", + "wasmparser 0.202.0", +] + +[[package]] +name = "wac-resolver" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c4c9482d68574aec10b72fa26429d0f91c5065ecbcf38a3146caf9f0bd7db" +dependencies = [ + "anyhow", + "futures", + "indexmap 2.7.1", + "log", + "miette 7.2.0", + "semver", + "thiserror 1.0.69", + "tokio", + "wac-parser", + "wac-types", + "warg-client", + "warg-crypto", + "warg-protocol", + "wit-component 0.202.0", +] + [[package]] name = "wac-types" version = "0.6.1" @@ -11492,6 +11596,25 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "wit-component" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c836b1fd9932de0431c1758d8be08212071b6bba0151f7bac826dbc4312a2a9" +dependencies = [ + "anyhow", + "bitflags 2.6.0", + "indexmap 2.7.1", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.202.0", + "wasm-metadata 0.202.0", + "wasmparser 0.202.0", + "wit-parser 0.202.0", +] + [[package]] name = "wit-component" version = "0.224.1" @@ -11531,6 +11654,24 @@ dependencies = [ "wit-parser 0.229.0", ] +[[package]] +name = "wit-parser" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744237b488352f4f27bca05a10acb79474415951c450e52ebd0da784c1df2bcc" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.7.1", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.202.0", +] + [[package]] name = "wit-parser" version = "0.224.1" diff --git a/Cargo.toml b/Cargo.toml index dec3677e0f..e5d10f9fa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ spin-app = { path = "crates/app" } spin-build = { path = "crates/build" } spin-common = { path = "crates/common" } spin-doctor = { path = "crates/doctor" } +spin-environments = { path = "crates/environments" } spin-factor-outbound-networking = { path = "crates/factor-outbound-networking" } spin-http = { path = "crates/http" } spin-loader = { path = "crates/loader" } diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index aa698b566c..29eb585ab1 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -8,6 +8,7 @@ edition = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } spin-common = { path = "../common" } +spin-environments = { path = "../environments" } spin-manifest = { path = "../manifest" } subprocess = "0.2" terminal = { path = "../terminal" } diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index c122d183e6..2242536666 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -16,31 +16,87 @@ use subprocess::{Exec, Redirection}; use crate::manifest::component_build_configs; /// If present, run the build command of each component. -pub async fn build(manifest_file: &Path, component_ids: &[String]) -> Result<()> { - let (components, manifest_err) = - component_build_configs(manifest_file) - .await - .with_context(|| { - format!( - "Cannot read manifest file from {}", - quoted_path(manifest_file) - ) - })?; +pub async fn build( + manifest_file: &Path, + component_ids: &[String], + target_checks: TargetChecking, + cache_root: Option, +) -> Result<()> { + let build_info = component_build_configs(manifest_file) + .await + .with_context(|| { + format!( + "Cannot read manifest file from {}", + quoted_path(manifest_file) + ) + })?; let app_dir = parent_dir(manifest_file)?; - let build_result = build_components(component_ids, components, app_dir); + let build_result = build_components(component_ids, build_info.components(), &app_dir); - if let Some(e) = manifest_err { + // Emit any required warnings now, so that they don't bury any errors. + if let Some(e) = build_info.load_error() { + // The manifest had errors. We managed to attempt a build anyway, but we want to + // let the user know about them. terminal::warn!("The manifest has errors not related to the Wasm component build. Error details:\n{e:#}"); + // Checking deployment targets requires a healthy manifest (because trigger types etc.), + // if any of these were specified, warn they are being skipped. + let should_have_checked_targets = + target_checks.check() && build_info.has_deployment_targets(); + if should_have_checked_targets { + terminal::warn!( + "The manifest error(s) prevented Spin from checking the deployment targets." + ); + } + } + + // If the build failed, exit with an error at this point. + build_result?; + + let Some(manifest) = build_info.manifest() else { + // We can't proceed to checking (because that needs a full healthy manifest), and we've + // already emitted any necessary warning, so quit. + return Ok(()); + }; + + if target_checks.check() { + let application = spin_environments::ApplicationToValidate::new( + manifest.clone(), + manifest_file.parent().unwrap(), + ) + .await + .context("unable to load application for checking against deployment targets")?; + let target_validation = spin_environments::validate_application_against_environment_ids( + &application, + build_info.deployment_targets(), + cache_root.clone(), + &app_dir, + ) + .await + .context("unable to check if the application is compatible with deployment targets")?; + + if !target_validation.is_ok() { + for error in target_validation.errors() { + terminal::error!("{error}"); + } + anyhow::bail!("All components built successfully, but one or more was incompatible with one or more of the deployment targets."); + } } - build_result + Ok(()) +} + +/// Run all component build commands, using the default options (build all +/// components, perform target checking). We run a "default build" in several +/// places and this centralises the logic of what such a "default build" means. +pub async fn build_default(manifest_file: &Path, cache_root: Option) -> Result<()> { + build(manifest_file, &[], TargetChecking::Check, cache_root).await } fn build_components( component_ids: &[String], components: Vec, - app_dir: PathBuf, + app_dir: &Path, ) -> Result<(), anyhow::Error> { let components_to_build = if component_ids.is_empty() { components @@ -70,7 +126,7 @@ fn build_components( components_to_build .into_iter() - .map(|c| build_component(c, &app_dir)) + .map(|c| build_component(c, app_dir)) .collect::, _>>()?; terminal::step!("Finished", "building all Spin components"); @@ -159,6 +215,21 @@ fn construct_workdir(app_dir: &Path, workdir: Option>) -> Resul Ok(cwd) } +/// Specifies target environment checking behaviour +pub enum TargetChecking { + /// The build should check that all components are compatible with all target environments. + Check, + /// The build should not check target environments. + Skip, +} + +impl TargetChecking { + /// Should the build check target environments? + fn check(&self) -> bool { + matches!(self, Self::Check) + } +} + #[cfg(test)] mod tests { use super::*; @@ -171,6 +242,8 @@ mod tests { #[tokio::test] async fn can_load_even_if_trigger_invalid() { let bad_trigger_file = test_data_root().join("bad_trigger.toml"); - build(&bad_trigger_file, &[]).await.unwrap(); + build(&bad_trigger_file, &[], TargetChecking::Skip, None) + .await + .unwrap(); } } diff --git a/crates/build/src/manifest.rs b/crates/build/src/manifest.rs index 2fcd68dadb..db570e4145 100644 --- a/crates/build/src/manifest.rs +++ b/crates/build/src/manifest.rs @@ -4,37 +4,120 @@ use std::{collections::BTreeMap, path::Path}; use spin_manifest::{schema::v2, ManifestVersion}; +#[allow(clippy::large_enum_variant)] // only ever constructed once +pub enum ManifestBuildInfo { + Loadable { + components: Vec, + deployment_targets: Vec, + manifest: spin_manifest::schema::v2::AppManifest, + }, + Unloadable { + components: Vec, + has_deployment_targets: bool, + load_error: spin_manifest::Error, + }, +} + +impl ManifestBuildInfo { + pub fn components(&self) -> Vec { + match self { + Self::Loadable { components, .. } => components.clone(), + Self::Unloadable { components, .. } => components.clone(), + } + } + + pub fn load_error(&self) -> Option<&spin_manifest::Error> { + match self { + Self::Loadable { .. } => None, + Self::Unloadable { load_error, .. } => Some(load_error), + } + } + + pub fn deployment_targets(&self) -> &[spin_manifest::schema::v2::TargetEnvironmentRef] { + match self { + Self::Loadable { + deployment_targets, .. + } => deployment_targets, + Self::Unloadable { .. } => &[], + } + } + + pub fn has_deployment_targets(&self) -> bool { + match self { + Self::Loadable { + deployment_targets, .. + } => !deployment_targets.is_empty(), + Self::Unloadable { + has_deployment_targets, + .. + } => *has_deployment_targets, + } + } + + pub fn manifest(&self) -> Option<&spin_manifest::schema::v2::AppManifest> { + match self { + Self::Loadable { manifest, .. } => Some(manifest), + Self::Unloadable { .. } => None, + } + } +} + /// Returns a map of component IDs to [`v2::ComponentBuildConfig`]s for the /// given (v1 or v2) manifest path. If the manifest cannot be loaded, the /// function attempts fallback: if fallback succeeds, result is Ok but the load error /// is also returned via the second part of the return value tuple. -pub async fn component_build_configs( - manifest_file: impl AsRef, -) -> Result<(Vec, Option)> { +pub async fn component_build_configs(manifest_file: impl AsRef) -> Result { let manifest = spin_manifest::manifest_from_file(&manifest_file); match manifest { - Ok(manifest) => Ok((build_configs_from_manifest(manifest), None)), - Err(e) => fallback_load_build_configs(&manifest_file) - .await - .map(|bc| (bc, Some(e))), + Ok(mut manifest) => { + spin_manifest::normalize::normalize_manifest(&mut manifest); + let components = build_configs_from_manifest(&manifest); + let deployment_targets = deployment_targets_from_manifest(&manifest); + Ok(ManifestBuildInfo::Loadable { + components, + deployment_targets, + manifest, + }) + } + Err(load_error) => { + // The manifest didn't load, but the problem might not be build-affecting. + // Try to fall back by parsing out only the bits we need. And if something + // goes wrong with the fallback, give up and return the original manifest load + // error. + let Ok(components) = fallback_load_build_configs(&manifest_file).await else { + return Err(load_error.into()); + }; + let Ok(has_deployment_targets) = has_deployment_targets(&manifest_file).await else { + return Err(load_error.into()); + }; + Ok(ManifestBuildInfo::Unloadable { + components, + has_deployment_targets, + load_error, + }) + } } } fn build_configs_from_manifest( - mut manifest: spin_manifest::schema::v2::AppManifest, + manifest: &spin_manifest::schema::v2::AppManifest, ) -> Vec { - spin_manifest::normalize::normalize_manifest(&mut manifest); - manifest .components - .into_iter() + .iter() .map(|(id, c)| ComponentBuildInfo { id: id.to_string(), - build: c.build, + build: c.build.clone(), }) .collect() } +fn deployment_targets_from_manifest( + manifest: &spin_manifest::schema::v2::AppManifest, +) -> Vec { + manifest.application.targets.clone() +} + async fn fallback_load_build_configs( manifest_file: impl AsRef, ) -> Result> { @@ -57,7 +140,23 @@ async fn fallback_load_build_configs( }) } -#[derive(Deserialize)] +async fn has_deployment_targets(manifest_file: impl AsRef) -> Result { + let manifest_text = tokio::fs::read_to_string(manifest_file).await?; + Ok(match ManifestVersion::detect(&manifest_text)? { + ManifestVersion::V1 => false, + ManifestVersion::V2 => { + let table: toml::value::Table = toml::from_str(&manifest_text)?; + table + .get("application") + .and_then(|a| a.as_table()) + .and_then(|t| t.get("targets")) + .and_then(|arr| arr.as_array()) + .is_some_and(|arr| !arr.is_empty()) + } + }) +} + +#[derive(Clone, Deserialize)] pub struct ComponentBuildInfo { #[serde(default)] pub id: String, diff --git a/crates/compose/src/lib.rs b/crates/compose/src/lib.rs index 3728ee9346..93a0fa39cc 100644 --- a/crates/compose/src/lib.rs +++ b/crates/compose/src/lib.rs @@ -1,7 +1,7 @@ use anyhow::Context; use indexmap::IndexMap; use semver::Version; -use spin_app::locked::{self, InheritConfiguration, LockedComponent, LockedComponentDependency}; +use spin_app::locked::InheritConfiguration as LockedInheritConfiguration; use spin_common::{ui::quoted_path, url::parse_file_url}; use spin_serde::{DependencyName, KebabId}; use std::collections::BTreeMap; @@ -29,18 +29,72 @@ use wac_graph::{CompositionGraph, NodeId}; /// composition graph into a byte array and return it. pub async fn compose( loader: &L, - component: &LockedComponent, + component: &L::Component, ) -> Result, ComposeError> { Composer::new(loader).compose(component).await } +/// A Spin component dependency. This abstracts over the metadata associated with the +/// dependency. The abstraction allows both manifest and lockfile types to participate in composition. +#[async_trait::async_trait] +pub trait DependencyLike { + fn inherit(&self) -> InheritConfiguration; + fn export(&self) -> &Option; +} + +pub enum InheritConfiguration { + All, + Some(Vec), +} + +/// A Spin component. This abstracts over the list of dependencies for the component. +/// The abstraction allows both manifest and lockfile types to participate in composition. +#[async_trait::async_trait] +pub trait ComponentLike { + type Dependency: DependencyLike; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator; + fn id(&self) -> &str; +} + +#[async_trait::async_trait] +impl ComponentLike for spin_app::locked::LockedComponent { + type Dependency = spin_app::locked::LockedComponentDependency; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator { + self.dependencies.iter() + } + + fn id(&self) -> &str { + &self.id + } +} + +#[async_trait::async_trait] +impl DependencyLike for spin_app::locked::LockedComponentDependency { + fn inherit(&self) -> InheritConfiguration { + match &self.inherit { + LockedInheritConfiguration::All => InheritConfiguration::All, + LockedInheritConfiguration::Some(cfgs) => InheritConfiguration::Some(cfgs.clone()), + } + } + + fn export(&self) -> &Option { + &self.export + } +} + /// This trait is used to load component source code from a locked component source across various embdeddings. #[async_trait::async_trait] pub trait ComponentSourceLoader { - async fn load_component_source( - &self, - source: &locked::LockedComponentSource, - ) -> anyhow::Result>; + type Component: ComponentLike; + type Dependency: DependencyLike; + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result>; + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result>; } /// A ComponentSourceLoader that loads component sources from the filesystem. @@ -48,9 +102,21 @@ pub struct ComponentSourceLoaderFs; #[async_trait::async_trait] impl ComponentSourceLoader for ComponentSourceLoaderFs { - async fn load_component_source( - &self, - source: &locked::LockedComponentSource, + type Component = spin_app::locked::LockedComponent; + type Dependency = spin_app::locked::LockedComponentDependency; + + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { + Self::load_from_locked_source(&source.source).await + } + + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result> { + Self::load_from_locked_source(&source.source).await + } +} + +impl ComponentSourceLoaderFs { + async fn load_from_locked_source( + source: &spin_app::locked::LockedComponentSource, ) -> anyhow::Result> { let source = source .content @@ -129,19 +195,19 @@ struct Composer<'a, L> { } impl<'a, L: ComponentSourceLoader> Composer<'a, L> { - async fn compose(mut self, component: &LockedComponent) -> Result, ComposeError> { + async fn compose(mut self, component: &L::Component) -> Result, ComposeError> { let source = self .loader - .load_component_source(&component.source) + .load_component_source(component) .await .map_err(ComposeError::PrepareError)?; - if component.dependencies.is_empty() { + if component.dependencies().len() == 0 { return Ok(source); } let (world_id, instantiation_id) = self - .register_package(&component.id, None, source) + .register_package(component.id(), None, source) .map_err(ComposeError::PrepareError)?; let prepared = self.prepare_dependencies(world_id, component).await?; @@ -180,7 +246,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { async fn prepare_dependencies( &mut self, world_id: WorldId, - component: &LockedComponent, + component: &L::Component, ) -> Result, ComposeError> { let imports = self.graph.types()[world_id].imports.clone(); @@ -188,7 +254,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { let mut mappings: BTreeMap> = BTreeMap::new(); - for (dependency_name, dependency) in &component.dependencies { + for (dependency_name, dependency) in component.dependencies() { let mut matched = Vec::new(); for import_name in &import_keys { @@ -201,7 +267,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { if matched.is_empty() { return Err(ComposeError::UnmatchedDependencyName { - component_id: component.id.clone(), + component_id: component.id().to_owned(), dependency_name: dependency_name.clone(), }); } @@ -225,7 +291,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { if !conflicts.is_empty() { return Err(ComposeError::DependencyConflicts { - component_id: component.id.clone(), + component_id: component.id().to_owned(), conflicts: conflicts .into_iter() .map(|(import_name, infos)| { @@ -330,19 +396,16 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { async fn register_dependency( &mut self, dependency_name: DependencyName, - dependency: &LockedComponentDependency, + dependency: &L::Dependency, ) -> anyhow::Result { - let mut dependency_source = self - .loader - .load_component_source(&dependency.source) - .await?; + let mut dependency_source = self.loader.load_dependency_source(dependency).await?; let package_name = match &dependency_name { DependencyName::Package(name) => name.package.to_string(), DependencyName::Plain(name) => name.to_string(), }; - match &dependency.inherit { + match dependency.inherit() { InheritConfiguration::Some(configurations) => { if configurations.is_empty() { // Configuration inheritance is disabled, apply deny_all adapter @@ -363,7 +426,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { manifest_name: dependency_name, instantiation_id, world_id, - export_name: dependency.export.clone(), + export_name: dependency.export().clone(), }) } diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml new file mode 100644 index 0000000000..6585fa8494 --- /dev/null +++ b/crates/environments/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "spin-environments" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +async-trait = "0.1" +bytes = "1.1" +chrono = { workspace = true } +futures = "0.3" +futures-util = "0.3" +id-arena = "2" +indexmap = "2" +oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" } +semver = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +spin-common = { path = "../common" } +spin-componentize = { path = "../componentize" } +spin-compose = { path = "../compose" } +spin-loader = { path = "../loader" } +spin-manifest = { path = "../manifest" } +spin-serde = { path = "../serde" } +toml = { workspace = true } +tokio = { version = "1.23", features = ["fs"] } +tracing = { workspace = true } +wac-parser = "0.6.0" +wac-resolver = "0.6.0" +wac-types = "0.6.0" +wasm-pkg-client = { workspace = true } +wasmparser = { workspace = true } +wit-component = { workspace = true } +wit-parser = { workspace = true } + +[lints] +workspace = true diff --git a/crates/environments/src/environment.rs b/crates/environments/src/environment.rs new file mode 100644 index 0000000000..63f9c67ded --- /dev/null +++ b/crates/environments/src/environment.rs @@ -0,0 +1,196 @@ +use std::{collections::HashMap, path::Path}; + +use anyhow::Context; +use spin_common::ui::quoted_path; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +mod definition; +mod env_loader; +mod lockfile; + +use definition::WorldName; + +/// A fully realised deployment environment, e.g. Spin 2.7, +/// SpinKube 3.1, Fermyon Cloud. The `TargetEnvironment` provides a mapping +/// from the Spin trigger types supported in the environment to the Component Model worlds +/// supported by that trigger type. (A trigger type may support more than one world, +/// for example when it supports multiple versions of the Spin or WASI interfaces.) +pub struct TargetEnvironment { + name: String, + trigger_worlds: HashMap, + unknown_trigger: UnknownTrigger, +} + +impl TargetEnvironment { + /// Loads the specified list of environments. This fetches all required + /// environment definitions from their references, and then chases packages + /// references until the entire target environment is fully loaded. + /// The function also caches registry references in the application directory, + /// to avoid loading from the network when the app is validated again. + pub async fn load_all( + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, + ) -> anyhow::Result> { + env_loader::load_environments(env_ids, cache_root, app_dir).await + } + + /// The environment name for UI purposes + pub fn name(&self) -> &str { + &self.name + } + + /// Returns true if the given trigger type can run in this environment. + pub fn supports_trigger_type(&self, trigger_type: &TriggerType) -> bool { + self.unknown_trigger.allows(trigger_type) || self.trigger_worlds.contains_key(trigger_type) + } + + /// Lists all worlds supported for the given trigger type in this environment. + pub fn worlds(&self, trigger_type: &TriggerType) -> &CandidateWorlds { + self.trigger_worlds + .get(trigger_type) + .or_else(|| self.unknown_trigger.worlds()) + .unwrap_or(NO_WORLDS) + } +} + +/// How a `TargetEnvironment` should validate components associated with trigger types +/// not listed in the/ environment definition. This is used for best-effort validation in +/// extensible environments. +/// +/// For example, a "forgiving" definition of Spin CLI environment would +/// validate that components associated with `cron` or `sqs` triggers adhere +/// to the platform world, even though it cannot validate that the exports are correct +/// or that the plugins are installed or up to date. This can result in failure at +/// runtime, but that may be better than refusing to let cron jobs run! +/// +/// On the other hand, the SpinKube environment rejects unknown triggers +/// because SpinKube does not allow arbitrary triggers to be linked at +/// runtime: the set of triggers is static for a given version. +enum UnknownTrigger { + /// Components for unknown trigger types fail validation. + Deny, + /// Components for unknown trigger types pass validation if they + /// conform to (at least) one of the listed worlds. + Allow(CandidateWorlds), +} + +impl UnknownTrigger { + fn allows(&self, _trigger_type: &TriggerType) -> bool { + matches!(self, Self::Allow(_)) + } + + fn worlds(&self) -> Option<&CandidateWorlds> { + match self { + Self::Deny => None, + Self::Allow(cw) => Some(cw), + } + } +} + +/// The set of worlds that a particular trigger type (in a given environment) +/// can accept. For example, the Spin 3.2 CLI `http` trigger accepts various +/// versions of the `spin:up/http-trigger` world. +/// +/// A component will pass target validation if it conforms to +/// at least one of these worlds. +#[derive(Default)] +pub struct CandidateWorlds { + worlds: Vec, +} + +impl<'a> IntoIterator for &'a CandidateWorlds { + type Item = &'a CandidateWorld; + + type IntoIter = std::slice::Iter<'a, CandidateWorld>; + + fn into_iter(self) -> Self::IntoIter { + self.worlds.iter() + } +} + +const NO_WORLDS: &CandidateWorlds = &CandidateWorlds { worlds: vec![] }; + +/// A WIT world; specifically, a WIT world provided by a Spin host, against which +/// a component can be validated. +pub struct CandidateWorld { + world: WorldName, + package: wit_parser::Package, + package_bytes: Vec, +} + +impl std::fmt::Display for CandidateWorld { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.world.fmt(f) + } +} + +impl CandidateWorld { + /// Namespaced but unversioned package name (e.g. spin:up) + pub fn package_namespaced_name(&self) -> String { + format!("{}:{}", self.package.name.namespace, self.package.name.name) + } + + /// The package version for the environment package. + pub fn package_version(&self) -> Option<&semver::Version> { + self.package.name.version.as_ref() + } + + /// The Wasm-encoded bytes of the environment package. + pub fn package_bytes(&self) -> &[u8] { + &self.package_bytes + } + + fn from_package_bytes(world: &WorldName, bytes: Vec) -> anyhow::Result { + let decoded = wit_component::decode(&bytes) + .with_context(|| format!("Failed to decode package for environment {world}"))?; + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .with_context(|| { + format!("The {world} package is invalid (no package for decoded package ID)") + })? + .clone(); + + Ok(Self { + world: world.to_owned(), + package, + package_bytes: bytes, + }) + } + + fn from_decoded_wasm( + world: &WorldName, + source: &Path, + decoded: wit_parser::decoding::DecodedWasm, + ) -> anyhow::Result { + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .with_context(|| { + format!( + "The {} environment is invalid (no package for decoded package ID)", + quoted_path(source) + ) + })? + .clone(); + + let bytes = wit_component::encode(decoded.resolve(), package_id)?; + + Ok(Self { + world: world.to_owned(), + package, + package_bytes: bytes, + }) + } +} + +pub(super) fn is_versioned(env_id: &str) -> bool { + env_id.contains(':') +} + +pub type TriggerType = String; diff --git a/crates/environments/src/environment/definition.rs b/crates/environments/src/environment/definition.rs new file mode 100644 index 0000000000..80d2e40aad --- /dev/null +++ b/crates/environments/src/environment/definition.rs @@ -0,0 +1,125 @@ +//! Environment definition types and serialisation (TOML) formats +//! +//! This module does *not* cover loading those definitions from remote +//! sources, or materialising WIT packages from files or registry references - +//! only the types. + +use std::collections::HashMap; + +use anyhow::Context; + +/// An environment definition, usually deserialised from a TOML document. +/// Example: +/// +/// ```ignore +/// # spin-up.3.2.toml +/// [triggers] +/// http = ["spin:up/http-trigger@3.2.0", "spin:up/http-trigger-rc20231018@3.2.0"] +/// redis = ["spin:up/redis-trigger@3.2.0"] +/// ``` +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentDefinition { + triggers: HashMap>, + default: Option>, +} + +impl EnvironmentDefinition { + pub fn triggers(&self) -> &HashMap> { + &self.triggers + } + + pub fn default(&self) -> Option<&Vec> { + self.default.as_ref() + } +} + +/// A reference to a world in an [EnvironmentDefinition]. This is formed +/// of a fully qualified (ns:pkg/id) world name, optionally with +/// a location from which to get the package (a registry or WIT directory). +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(untagged, deny_unknown_fields)] +pub enum WorldRef { + DefaultRegistry(WorldName), + Registry { + registry: String, + world: WorldName, + }, + WitDirectory { + path: std::path::PathBuf, + world: WorldName, + }, +} + +/// The qualified name of a world, e.g. spin:up/http-trigger@3.2.0. +/// +/// (Internally it is represented as a PackageName plus unqualified +/// world name, but it stringises to the standard WIT qualified name.) +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(try_from = "String")] +pub struct WorldName { + package: wit_parser::PackageName, + world: String, +} + +impl WorldName { + pub fn package(&self) -> &wit_parser::PackageName { + &self.package + } + + pub fn package_namespaced_name(&self) -> String { + format!("{}:{}", self.package.namespace, self.package.name) + } + + pub fn package_ref(&self) -> anyhow::Result { + let pkg_name = self.package_namespaced_name(); + pkg_name + .parse() + .with_context(|| format!("Environment {pkg_name} is not a valid package name")) + } + + pub fn package_version(&self) -> Option<&semver::Version> { + self.package.version.as_ref() + } +} + +impl TryFrom for WorldName { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + use wasmparser::names::{ComponentName, ComponentNameKind}; + + // World qnames have the same syntactic form as interface qnames + let parsed = ComponentName::new(&value, 0)?; + let ComponentNameKind::Interface(itf) = parsed.kind() else { + anyhow::bail!("{value} is not a well-formed world name"); + }; + + let package = wit_parser::PackageName { + namespace: itf.namespace().to_string(), + name: itf.package().to_string(), + version: itf.version(), + }; + + let world = itf.interface().to_string(); + + Ok(Self { package, world }) + } +} + +impl std::fmt::Display for WorldName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.package.namespace)?; + f.write_str(":")?; + f.write_str(&self.package.name)?; + f.write_str("/")?; + f.write_str(&self.world)?; + + if let Some(v) = self.package.version.as_ref() { + f.write_str("@")?; + f.write_str(&v.to_string())?; + } + + Ok(()) + } +} diff --git a/crates/environments/src/environment/env_loader.rs b/crates/environments/src/environment/env_loader.rs new file mode 100644 index 0000000000..2433ae4722 --- /dev/null +++ b/crates/environments/src/environment/env_loader.rs @@ -0,0 +1,314 @@ +//! Loading target environments, from a list of references through to +//! a fully realised collection of WIT packages with their worlds and +//! mappings. + +use std::{collections::HashMap, path::Path}; + +use anyhow::Context; +use futures::future::try_join_all; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +use super::definition::{EnvironmentDefinition, WorldName, WorldRef}; +use super::lockfile::TargetEnvironmentLockfile; +use super::{is_versioned, CandidateWorld, CandidateWorlds, TargetEnvironment, UnknownTrigger}; + +const DEFAULT_ENV_DEF_REGISTRY: &str = "ghcr.io/itowlson/envs"; +const DEFAULT_PACKAGE_REGISTRY: &str = "spinframework.dev"; + +/// Load all the listed environments from their registries or paths. +/// Registry data will be cached, with a lockfile under `.spin` mapping +/// environment IDs to digests (to allow cache lookup without needing +/// to fetch the digest from the registry). +pub async fn load_environments( + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, +) -> anyhow::Result> { + if env_ids.is_empty() { + return Ok(Default::default()); + } + + let cache = spin_loader::cache::Cache::new(cache_root) + .await + .context("Unable to create cache")?; + let lockfile_dir = app_dir.join(".spin"); + let lockfile_path = lockfile_dir.join("target-environments.lock"); + + let orig_lockfile: TargetEnvironmentLockfile = tokio::fs::read_to_string(&lockfile_path) + .await + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + let lockfile = std::sync::Arc::new(tokio::sync::RwLock::new(orig_lockfile.clone())); + + let envs = try_join_all( + env_ids + .iter() + .map(|e| load_environment(e, &cache, &lockfile)), + ) + .await?; + + let final_lockfile = &*lockfile.read().await; + if *final_lockfile != orig_lockfile { + if let Ok(lockfile_json) = serde_json::to_string_pretty(&final_lockfile) { + _ = tokio::fs::create_dir_all(lockfile_dir).await; + _ = tokio::fs::write(&lockfile_path, lockfile_json).await; // failure to update lockfile is not an error + } + } + + Ok(envs) +} + +/// Loads the given `TargetEnvironment` from a registry or directory. +async fn load_environment( + env_id: &TargetEnvironmentRef, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + match env_id { + TargetEnvironmentRef::DefaultRegistry(id) => { + load_environment_from_registry(DEFAULT_ENV_DEF_REGISTRY, id, cache, lockfile).await + } + TargetEnvironmentRef::Registry { registry, id } => { + load_environment_from_registry(registry, id, cache, lockfile).await + } + TargetEnvironmentRef::File { path } => { + load_environment_from_file(path, cache, lockfile).await + } + } +} + +/// Loads the given `TargetEnvironment` from the given registry, or +/// from cache if available. If the environment is not in cache, the +/// encoded WIT will be cached, and the in-memory lockfile object +/// updated. +async fn load_environment_from_registry( + registry: &str, + env_id: &str, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let env_def_toml = load_env_def_toml_from_registry(registry, env_id, cache, lockfile).await?; + load_environment_from_toml(env_id, &env_def_toml, cache, lockfile).await +} + +async fn load_environment_from_file( + path: &Path, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_owned()) + .unwrap(); + let toml_text = tokio::fs::read_to_string(path).await?; + load_environment_from_toml(&name, &toml_text, cache, lockfile).await +} + +async fn load_environment_from_toml( + name: &str, + toml_text: &str, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let env: EnvironmentDefinition = toml::from_str(toml_text)?; + + let mut trigger_worlds = HashMap::new(); + + // TODO: parallel all the things + // TODO: this loads _all_ triggers not just the ones we need + for (trigger_type, world_refs) in env.triggers() { + trigger_worlds.insert( + trigger_type.to_owned(), + load_worlds(world_refs, cache, lockfile).await?, + ); + } + + let unknown_trigger = match env.default() { + None => UnknownTrigger::Deny, + Some(world_refs) => UnknownTrigger::Allow(load_worlds(world_refs, cache, lockfile).await?), + }; + + Ok(TargetEnvironment { + name: name.to_owned(), + trigger_worlds, + unknown_trigger, + }) +} + +async fn load_env_def_toml_from_registry( + registry: &str, + env_id: &str, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + if let Some(digest) = lockfile.read().await.env_digest(registry, env_id) { + if let Ok(cache_file) = cache.data_file(digest) { + if let Ok(bytes) = tokio::fs::read(&cache_file).await { + return Ok(String::from_utf8_lossy(&bytes).to_string()); + } + } + } + + let (bytes, digest) = download_env_def_file(registry, env_id).await?; + + let toml_text = String::from_utf8_lossy(&bytes).to_string(); + + _ = cache.write_data(bytes, &digest).await; + lockfile + .write() + .await + .set_env_digest(registry, env_id, &digest); + + Ok(toml_text) +} + +async fn download_env_def_file(registry: &str, env_id: &str) -> anyhow::Result<(Vec, String)> { + // This implies env_id is in the format spin-up:3.2 which WHO KNOWS + let registry_id = if is_versioned(env_id) { + env_id.to_string() + } else { + // Testing versionless tags with GHCR it didn't work + // TODO: is this expected or am I being a dolt + // TODO: is this a suitable workaround + format!("{env_id}:latest") + }; + + let reference = format!("{registry}/{registry_id}"); + let reference = oci_distribution::Reference::try_from(reference)?; + + let config = oci_distribution::client::ClientConfig::default(); + let client = oci_distribution::client::Client::new(config); + let auth = oci_distribution::secrets::RegistryAuth::Anonymous; + + let (manifest, digest) = client.pull_manifest(&reference, &auth).await?; + + let im = match manifest { + oci_distribution::manifest::OciManifest::Image(im) => im, + oci_distribution::manifest::OciManifest::ImageIndex(_ind) => { + anyhow::bail!("found image index instead of image manifest, get in the sea") + } + }; + + let count = im.layers.len(); + + if count != 1 { + anyhow::bail!("artifact {reference} should have had exactly one layer"); + } + + let the_layer = &im.layers[0]; + let mut out = Vec::with_capacity(the_layer.size.try_into().unwrap_or_default()); + client.pull_blob(&reference, the_layer, &mut out).await?; + + Ok((out, digest)) +} + +async fn load_worlds( + world_refs: &[WorldRef], + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let mut worlds = vec![]; + + for world_ref in world_refs { + worlds.push(load_world(world_ref, cache, lockfile).await?); + } + + Ok(CandidateWorlds { worlds }) +} + +async fn load_world( + world_ref: &WorldRef, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + match world_ref { + WorldRef::DefaultRegistry(world) => { + load_world_from_registry(DEFAULT_PACKAGE_REGISTRY, world, cache, lockfile).await + } + WorldRef::Registry { registry, world } => { + load_world_from_registry(registry, world, cache, lockfile).await + } + WorldRef::WitDirectory { path, world } => load_world_from_dir(path, world), + } +} + +fn load_world_from_dir(path: &Path, world: &WorldName) -> anyhow::Result { + let mut resolve = wit_parser::Resolve::default(); + let (pkg_id, _) = resolve.push_dir(path)?; + let decoded = wit_parser::decoding::DecodedWasm::WitPackage(resolve, pkg_id); + CandidateWorld::from_decoded_wasm(world, path, decoded) +} + +/// Loads the given `TargetEnvironment` from the given registry, or +/// from cache if available. If the environment is not in cache, the +/// encoded WIT will be cached, and the in-memory lockfile object +/// updated. +async fn load_world_from_registry( + registry: &str, + world_name: &WorldName, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + use futures_util::TryStreamExt; + + // TODO: the lookup should presumably be a package name + if let Some(digest) = lockfile + .read() + .await + .package_digest(registry, world_name.package()) + { + if let Ok(cache_file) = cache.wasm_file(digest) { + if let Ok(bytes) = tokio::fs::read(&cache_file).await { + return CandidateWorld::from_package_bytes(world_name, bytes); + } + } + } + + let pkg_name = world_name.package_namespaced_name(); + let pkg_ref = world_name.package_ref()?; + + let wkg_registry: wasm_pkg_client::Registry = registry + .parse() + .with_context(|| format!("Registry {registry} is not a valid registry name"))?; + + // TODO: this requires wkg configuration which shouldn't be on users: + // is there a better way to handle it? + let mut wkg_config = wasm_pkg_client::Config::global_defaults().await?; + wkg_config.set_package_registry_override( + pkg_ref, + wasm_pkg_client::RegistryMapping::Registry(wkg_registry), + ); + + let client = wasm_pkg_client::Client::new(wkg_config); + + let package = pkg_name + .to_owned() + .try_into() + .with_context(|| format!("Failed to parse environment name {pkg_name} as package name"))?; + let version = world_name.package_version().unwrap(); // TODO: surely we can cope with unversioned? surely? + + let release = client + .get_release(&package, version) + .await + .with_context(|| format!("Failed to get {} from registry", world_name.package()))?; + let stm = client + .stream_content(&package, &release) + .await + .with_context(|| format!("Failed to get {} from registry", world_name.package()))?; + let bytes = stm + .try_collect::() + .await + .with_context(|| format!("Failed to get {} from registry", world_name.package()))? + .to_vec(); + + let digest = release.content_digest.to_string(); + _ = cache.write_wasm(&bytes, &digest).await; // Failure to cache is not fatal + lockfile + .write() + .await + .set_package_digest(registry, world_name.package(), &digest); + + CandidateWorld::from_package_bytes(world_name, bytes) +} diff --git a/crates/environments/src/environment/lockfile.rs b/crates/environments/src/environment/lockfile.rs new file mode 100644 index 0000000000..d638e458b7 --- /dev/null +++ b/crates/environments/src/environment/lockfile.rs @@ -0,0 +1,124 @@ +use std::collections::HashMap; + +use super::is_versioned; + +const DIGEST_TTL_HOURS: i64 = 24; + +/// Serialisation format for the lockfile: registry -> env|pkg -> { name -> digest } +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TargetEnvironmentLockfile(HashMap); + +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct Digests { + env: HashMap, + package: HashMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +enum ExpirableDigest { + Forever(String), + Expiring { + digest: String, + correct_at: chrono::DateTime, + }, +} + +impl TargetEnvironmentLockfile { + pub fn env_digest(&self, registry: &str, env_id: &str) -> Option<&str> { + self.0 + .get(registry) + .and_then(|ds| ds.env.get(env_id)) + .and_then(|s| s.current()) + } + + pub fn set_env_digest(&mut self, registry: &str, env_id: &str, digest: &str) { + // If the environment is versioned, we assume it will not change (that is, any changes will + // be reflected as a new version). If the environment is *not* versioned, it represents + // a hosted service which may change over time: allow the cached definition to expire every day or + // so that we do not use a definition that is out of sync with the actual service. + let expirable_digest = if is_versioned(env_id) { + ExpirableDigest::forever(digest) + } else { + ExpirableDigest::expiring(digest) + }; + + match self.0.get_mut(registry) { + Some(ds) => { + ds.env.insert(env_id.to_string(), expirable_digest); + } + None => { + let map = vec![(env_id.to_string(), expirable_digest)] + .into_iter() + .collect(); + let ds = Digests { + env: map, + package: Default::default(), + }; + self.0.insert(registry.to_string(), ds); + } + } + } + + pub fn package_digest( + &self, + registry: &str, + package: &wit_parser::PackageName, + ) -> Option<&str> { + self.0 + .get(registry) + .and_then(|ds| ds.package.get(&package.to_string())) + .map(|s| s.as_str()) + } + + pub fn set_package_digest( + &mut self, + registry: &str, + package: &wit_parser::PackageName, + digest: &str, + ) { + match self.0.get_mut(registry) { + Some(ds) => { + ds.package.insert(package.to_string(), digest.to_string()); + } + None => { + let map = vec![(package.to_string(), digest.to_string())] + .into_iter() + .collect(); + let ds = Digests { + env: Default::default(), + package: map, + }; + self.0.insert(registry.to_string(), ds); + } + } + } +} + +impl ExpirableDigest { + fn current(&self) -> Option<&str> { + match self { + Self::Forever(digest) => Some(digest), + Self::Expiring { digest, correct_at } => { + let now = chrono::Utc::now(); + let time_since = now - correct_at; + if time_since.abs().num_hours() > DIGEST_TTL_HOURS { + None + } else { + Some(digest) + } + } + } + } + + fn forever(digest: &str) -> Self { + Self::Forever(digest.to_string()) + } + + fn expiring(digest: &str) -> Self { + Self::Expiring { + digest: digest.to_string(), + correct_at: chrono::Utc::now(), + } + } +} diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs new file mode 100644 index 0000000000..748aba9a23 --- /dev/null +++ b/crates/environments/src/lib.rs @@ -0,0 +1,227 @@ +use anyhow::{anyhow, Context}; + +mod environment; +mod loader; + +use environment::{CandidateWorld, CandidateWorlds, TargetEnvironment, TriggerType}; +pub use loader::ApplicationToValidate; +use loader::ComponentToValidate; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +/// The result of validating an application against a list of target environments. +/// If `is_ok` returns true (or equivalently if the `errors` collection is empty), +/// the application passed validation, and can run in all the environments against +/// which it was checked. Otherwise, at least one component cannot run in at least +/// one target environment, and the `errors` collection contains the details. +#[derive(Default)] +pub struct TargetEnvironmentValidation(Vec); + +impl TargetEnvironmentValidation { + pub fn is_ok(&self) -> bool { + self.0.is_empty() + } + + pub fn errors(&self) -> &[anyhow::Error] { + &self.0 + } +} + +/// Validates *all* application components against the list of referenced target enviroments. Each component must conform +/// to *all* environments to pass. +/// +/// If the return value is `Ok(...)`, this means only that we were able to perform the validation. +/// The caller **MUST** still check the returned [TargetEnvironmentValidation] to determine the +/// outcome of validation. +/// +/// If the return value is `Err(...)`, then we weren't able even to attempt validation. +pub async fn validate_application_against_environment_ids( + application: &ApplicationToValidate, + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, +) -> anyhow::Result { + if env_ids.is_empty() { + return Ok(Default::default()); + } + + let envs = TargetEnvironment::load_all(env_ids, cache_root, app_dir).await?; + validate_application_against_environments(application, &envs).await +} + +/// Validates *all* application components against the list of (realised) target enviroments. Each component must conform +/// to *all* environments to pass. +/// +/// For the slightly funky return type, see [validate_application_against_environment_ids]. +async fn validate_application_against_environments( + application: &ApplicationToValidate, + envs: &[TargetEnvironment], +) -> anyhow::Result { + for trigger_type in application.trigger_types() { + if let Some(env) = envs.iter().find(|e| !e.supports_trigger_type(trigger_type)) { + anyhow::bail!( + "Environment {} does not support trigger type {trigger_type}", + env.name() + ); + } + } + + let components_by_trigger_type = application.components_by_trigger_type().await?; + + let mut errs = vec![]; + + for (trigger_type, component) in components_by_trigger_type { + for component in &component { + errs.extend( + validate_component_against_environments(envs, &trigger_type, component).await, + ); + } + } + + Ok(TargetEnvironmentValidation(errs)) +} + +/// Validates the component against the list of target enviroments. The component must conform +/// to *all* environments to pass. +/// +/// The return value contains the list of validation errors. There may be up to one error per +/// target environment, explaining why the component cannot run in that environment. +/// An empty list means the component has passed validation and is compatible with +/// all target environments. +async fn validate_component_against_environments( + envs: &[TargetEnvironment], + trigger_type: &TriggerType, + component: &ComponentToValidate<'_>, +) -> Vec { + let mut errs = vec![]; + + for env in envs { + let worlds = env.worlds(trigger_type); + if let Some(e) = validate_wasm_against_any_world(env, worlds, component) + .await + .err() + { + errs.push(e); + } + } + + if errs.is_empty() { + tracing::info!( + "Validated component {} {} against all target worlds", + component.id(), + component.source_description() + ); + } + + errs +} + +/// Validates the component against the list of candidate worlds. The component must conform +/// to *at least one* candidate world to pass (since if it can run in one world provided by +/// the target environment, it can run in the target environment). +async fn validate_wasm_against_any_world( + env: &TargetEnvironment, + worlds: &CandidateWorlds, + component: &ComponentToValidate<'_>, +) -> anyhow::Result<()> { + let mut result = Ok(()); + for target_world in worlds { + tracing::debug!( + "Trying component {} {} against target world {target_world}", + component.id(), + component.source_description(), + ); + match validate_wasm_against_world(env, target_world, component).await { + Ok(()) => { + tracing::info!( + "Validated component {} {} against target world {target_world}", + component.id(), + component.source_description(), + ); + return Ok(()); + } + Err(e) => { + // Record the error, but continue in case a different world succeeds + tracing::info!( + "Rejecting component {} {} for target world {target_world} because {e:?}", + component.id(), + component.source_description(), + ); + result = Err(e); + } + } + } + result +} + +async fn validate_wasm_against_world( + env: &TargetEnvironment, + target_world: &CandidateWorld, + component: &ComponentToValidate<'_>, +) -> anyhow::Result<()> { + // Because we are abusing a composition tool to do validation, we have to + // provide a name by which to refer to the component in the dummy composition. + let component_name = "root:component"; + let component_key = wac_types::BorrowedPackageKey::from_name_and_version(component_name, None); + + // wac is going to get the world from the environment package bytes. + // This constructs a key for that mapping. + let env_pkg_name = target_world.package_namespaced_name(); + let env_pkg_key = wac_types::BorrowedPackageKey::from_name_and_version( + &env_pkg_name, + target_world.package_version(), + ); + + let env_name = env.name(); + + let wac_text = format!( + r#" + package validate:component@1.0.0 targets {target_world}; + let c = new {component_name} {{ ... }}; + export c...; + "# + ); + + let doc = wac_parser::Document::parse(&wac_text) + .context("Internal error constructing WAC document for target checking")?; + + // TODO: if we end up needing the registry, we need to do this dance + // for things we are providing separately, or the registry will try to + // hoover them up and will fail. + // let mut refpkgs = wac_resolver::packages(&doc)?; + // refpkgs.shift_remove(&env_pkg_key); + // refpkgs.shift_remove(&component_key); + + // TODO: determine if this is needed in circumstances other than the simple test + // let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?; + // let mut packages = reg_resolver + // .resolve(&refpkgs) + // .await + // .context("reg_resolver.resolve failed")?; + + let mut packages: indexmap::IndexMap> = + Default::default(); + + packages.insert(env_pkg_key, target_world.package_bytes().to_vec()); + packages.insert(component_key, component.wasm_bytes().to_vec()); + + match doc.resolve(packages) { + Ok(_) => Ok(()), + Err(wac_parser::resolution::Error::TargetMismatch { kind, name, world, .. }) => { + // This one doesn't seem to get hit at the moment - we get MissingTargetExport or ImportNotInTarget instead + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} expects an {} named {name}", component.id(), component.source_description(), kind.to_string().to_lowercase())) + } + Err(wac_parser::resolution::Error::MissingTargetExport { name, world, .. }) => { + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} requires an export named {name}, which the component does not provide", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::PackageMissingExport { export, .. }) => { + // TODO: The export here seems wrong - it seems to contain the world name rather than the interface name + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_world} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::ImportNotInTarget { name, world, .. }) => { + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} does not provide an import named {name}, which the component requires", component.id(), component.source_description())) + } + Err(e) => { + Err(anyhow!(e)) + }, + } +} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs new file mode 100644 index 0000000000..274b368bfe --- /dev/null +++ b/crates/environments/src/loader.rs @@ -0,0 +1,246 @@ +//! Loading an application for validation. + +use std::path::Path; + +use anyhow::{anyhow, Context}; +use futures::future::try_join_all; +use spin_common::ui::quoted_path; + +pub(crate) struct ComponentToValidate<'a> { + id: &'a str, + source_description: String, + wasm: Vec, +} + +impl ComponentToValidate<'_> { + pub fn id(&self) -> &str { + self.id + } + + pub fn source_description(&self) -> &str { + &self.source_description + } + + pub fn wasm_bytes(&self) -> &[u8] { + &self.wasm + } +} + +pub struct ApplicationToValidate { + manifest: spin_manifest::schema::v2::AppManifest, + wasm_loader: spin_loader::WasmLoader, +} + +impl ApplicationToValidate { + pub async fn new( + manifest: spin_manifest::schema::v2::AppManifest, + base_dir: impl AsRef, + ) -> anyhow::Result { + let wasm_loader = + spin_loader::WasmLoader::new(base_dir.as_ref().to_owned(), None, None).await?; + Ok(Self { + manifest, + wasm_loader, + }) + } + + fn component_source<'a>( + &'a self, + trigger: &'a spin_manifest::schema::v2::Trigger, + ) -> anyhow::Result> { + let component_spec = trigger + .component + .as_ref() + .ok_or_else(|| anyhow!("No component specified for trigger {}", trigger.id))?; + let (id, source, dependencies) = match component_spec { + spin_manifest::schema::v2::ComponentSpec::Inline(c) => { + (trigger.id.as_str(), &c.source, &c.dependencies) + } + spin_manifest::schema::v2::ComponentSpec::Reference(r) => { + let id = r.as_ref(); + let Some(component) = self.manifest.components.get(r) else { + anyhow::bail!( + "Component {id} specified for trigger {} does not exist", + trigger.id + ); + }; + (id, &component.source, &component.dependencies) + } + }; + + Ok(ComponentSource { + id, + source, + dependencies: WrappedComponentDependencies::new(dependencies), + }) + } + + pub fn trigger_types(&self) -> impl Iterator { + self.manifest.triggers.keys() + } + + pub fn triggers( + &self, + ) -> impl Iterator)> { + self.manifest.triggers.iter() + } + + pub(crate) async fn components_by_trigger_type( + &self, + ) -> anyhow::Result>)>> { + use futures::FutureExt; + + let components_by_trigger_type_futs = self.triggers().map(|(ty, ts)| { + self.components_for_trigger(ts) + .map(|css| css.map(|css| (ty.to_owned(), css))) + }); + let components_by_trigger_type = try_join_all(components_by_trigger_type_futs) + .await + .context("Failed to prepare components for target environment checking")?; + Ok(components_by_trigger_type) + } + + async fn components_for_trigger<'a>( + &'a self, + triggers: &'a [spin_manifest::schema::v2::Trigger], + ) -> anyhow::Result>> { + let component_futures = triggers.iter().map(|t| self.load_and_resolve_trigger(t)); + try_join_all(component_futures).await + } + + async fn load_and_resolve_trigger<'a>( + &'a self, + trigger: &'a spin_manifest::schema::v2::Trigger, + ) -> anyhow::Result> { + let component = self.component_source(trigger)?; + + let loader = ComponentSourceLoader::new(&self.wasm_loader); + + let wasm = spin_compose::compose(&loader, &component).await.with_context(|| format!("Spin needed to compose dependencies for {} as part of target checking, but composition failed", component.id))?; + + Ok(ComponentToValidate { + id: component.id, + source_description: source_description(component.source), + wasm, + }) + } +} + +struct ComponentSource<'a> { + id: &'a str, + source: &'a spin_manifest::schema::v2::ComponentSource, + dependencies: WrappedComponentDependencies, +} + +struct ComponentSourceLoader<'a> { + wasm_loader: &'a spin_loader::WasmLoader, +} + +impl<'a> ComponentSourceLoader<'a> { + pub fn new(wasm_loader: &'a spin_loader::WasmLoader) -> Self { + Self { wasm_loader } + } +} + +#[async_trait::async_trait] +impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> { + type Component = ComponentSource<'a>; + type Dependency = WrappedComponentDependency; + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { + let path = self + .wasm_loader + .load_component_source(source.id, source.source) + .await?; + let bytes = tokio::fs::read(&path).await?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + Ok(component.into()) + } + + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result> { + let (path, _) = self + .wasm_loader + .load_component_dependency(&source.name, &source.dependency) + .await?; + let bytes = tokio::fs::read(&path).await?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + Ok(component.into()) + } +} + +// This exists only to thwart the orphan rule +struct WrappedComponentDependency { + name: spin_serde::DependencyName, + dependency: spin_manifest::schema::v2::ComponentDependency, +} + +// To manage lifetimes around the thwarting of the orphan rule +struct WrappedComponentDependencies { + dependencies: indexmap::IndexMap, +} + +impl WrappedComponentDependencies { + fn new(deps: &spin_manifest::schema::v2::ComponentDependencies) -> Self { + let dependencies = deps + .inner + .clone() + .into_iter() + .map(|(k, v)| { + ( + k.clone(), + WrappedComponentDependency { + name: k, + dependency: v, + }, + ) + }) + .collect(); + Self { dependencies } + } +} + +#[async_trait::async_trait] +impl spin_compose::ComponentLike for ComponentSource<'_> { + type Dependency = WrappedComponentDependency; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator + { + self.dependencies.dependencies.iter() + } + + fn id(&self) -> &str { + self.id + } +} + +#[async_trait::async_trait] +impl spin_compose::DependencyLike for WrappedComponentDependency { + fn inherit(&self) -> spin_compose::InheritConfiguration { + // We don't care because this never runs - it is only used to + // verify import satisfaction. Choosing All avoids the compose + // algorithm meddling with it using the deny adapter. + spin_compose::InheritConfiguration::All + } + + fn export(&self) -> &Option { + match &self.dependency { + spin_manifest::schema::v2::ComponentDependency::Version(_) => &None, + spin_manifest::schema::v2::ComponentDependency::Package { export, .. } => export, + spin_manifest::schema::v2::ComponentDependency::Local { export, .. } => export, + spin_manifest::schema::v2::ComponentDependency::HTTP { export, .. } => export, + } + } +} + +fn source_description(source: &spin_manifest::schema::v2::ComponentSource) -> String { + match source { + spin_manifest::schema::v2::ComponentSource::Local(path) => { + format!("file {}", quoted_path(path)) + } + spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), + spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { + format!("package {package}") + } + } +} diff --git a/crates/loader/src/lib.rs b/crates/loader/src/lib.rs index ea64bac40d..0142425bfd 100644 --- a/crates/loader/src/lib.rs +++ b/crates/loader/src/lib.rs @@ -23,6 +23,8 @@ mod fs; mod http; mod local; +pub use local::WasmLoader; + /// Maximum number of files to copy (or download) concurrently pub(crate) const MAX_FILE_LOADING_CONCURRENCY: usize = 16; diff --git a/crates/loader/src/local.rs b/crates/loader/src/local.rs index 1168330be7..0cac194202 100644 --- a/crates/loader/src/local.rs +++ b/crates/loader/src/local.rs @@ -23,8 +23,8 @@ use crate::{cache::Cache, FilesMountStrategy}; pub struct LocalLoader { app_root: PathBuf, files_mount_strategy: FilesMountStrategy, - cache: Cache, - file_loading_permits: Semaphore, + file_loading_permits: std::sync::Arc, + wasm_loader: WasmLoader, } impl LocalLoader { @@ -35,12 +35,14 @@ impl LocalLoader { ) -> Result { let app_root = safe_canonicalize(app_root) .with_context(|| format!("Invalid manifest dir `{}`", app_root.display()))?; + let file_loading_permits = + std::sync::Arc::new(Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY)); Ok(Self { - app_root, + app_root: app_root.clone(), files_mount_strategy, - cache: Cache::new(cache_root).await?, // Limit concurrency to avoid hitting system resource limits - file_loading_permits: Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY), + file_loading_permits: file_loading_permits.clone(), + wasm_loader: WasmLoader::new(app_root, cache_root, Some(file_loading_permits)).await?, }) } @@ -259,74 +261,15 @@ impl LocalLoader { dependency_name: DependencyName, dependency: v2::ComponentDependency, ) -> Result { - let (content, export) = match dependency { - v2::ComponentDependency::Version(version) => { - let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; - - // This `unwrap()` should be OK because we've already validated - // this form of dependency requires a package name, i.e. the - // dependency name is not a kebab id. - let package = dependency_name.package().unwrap(); - - let content = self.load_registry_source(None, package, &version).await?; - (content, None) - } - v2::ComponentDependency::Package { - version, - registry, - package, - export, - } => { - let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; - - let package = match package { - Some(package) => { - package.parse().with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid package name ({package:?})"))? - } - None => { - // This `unwrap()` should be OK because we've already validated - // this form of dependency requires a package name, i.e. the - // dependency name is not a kebab id. - dependency_name - .package() - .cloned() - .unwrap() - } - }; - - let registry = match registry { - Some(registry) => { - registry - .parse() - .map(Some) - .with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid registry name ({registry:?})"))? - } - None => None, - }; - - let content = self - .load_registry_source(registry.as_ref(), &package, &version) - .await?; - (content, export) - } - v2::ComponentDependency::Local { path, export } => { - let content = file_content_ref(self.app_root.join(path))?; - (content, export) - } - v2::ComponentDependency::HTTP { - url, - digest, - export, - } => { - let content = self.load_http_source(&url, &digest).await?; - (content, export) - } - }; + let (content, export) = self + .wasm_loader + .load_component_dependency(&dependency_name, &dependency) + .await?; Ok(LockedComponentDependency { source: LockedComponentSource { content_type: "application/wasm".into(), - content, + content: file_content_ref(content)?, }, export, inherit: if inherit_configuration { @@ -344,116 +287,16 @@ impl LocalLoader { component_id: &KebabId, source: v2::ComponentSource, ) -> Result { - let content = match source { - v2::ComponentSource::Local(path) => file_content_ref(self.app_root.join(path))?, - v2::ComponentSource::Remote { url, digest } => { - self.load_http_source(&url, &digest).await? - } - v2::ComponentSource::Registry { - registry, - package, - version, - } => { - let version = semver::Version::parse(&version).with_context(|| format!("Component {component_id} specifies an invalid semantic version ({version:?}) for its package version"))?; - let version_req = format!("={version}").parse().expect("version"); - - self.load_registry_source(registry.as_ref(), &package, &version_req) - .await? - } - }; + let path = self + .wasm_loader + .load_component_source(component_id.as_ref(), &source) + .await?; Ok(LockedComponentSource { content_type: "application/wasm".into(), - content, + content: file_content_ref(path)?, }) } - // Load a Wasm source from the given HTTP ContentRef source URL and - // return a ContentRef an absolute path to the local copy. - async fn load_http_source(&self, url: &str, digest: &str) -> Result { - ensure!( - digest.starts_with("sha256:"), - "invalid `digest` {digest:?}; must start with 'sha256:'" - ); - let path = if let Ok(cached_path) = self.cache.wasm_file(digest) { - cached_path - } else { - let _loading_permit = self.file_loading_permits.acquire().await?; - - self.cache.ensure_dirs().await?; - let dest = self.cache.wasm_path(digest); - verified_download( - url, - digest, - &dest, - crate::http::DestinationConvention::ContentIndexed, - ) - .await - .with_context(|| format!("Error fetching source URL {url:?}"))?; - dest - }; - file_content_ref(path) - } - - async fn load_registry_source( - &self, - registry: Option<&wasm_pkg_client::Registry>, - package: &wasm_pkg_client::PackageRef, - version: &semver::VersionReq, - ) -> Result { - let mut client_config = wasm_pkg_client::Config::global_defaults().await?; - - if let Some(registry) = registry.cloned() { - let mapping = wasm_pkg_client::RegistryMapping::Registry(registry); - client_config.set_package_registry_override(package.clone(), mapping); - } - let pkg_loader = wasm_pkg_client::Client::new(client_config); - - let mut releases = pkg_loader.list_all_versions(package).await.map_err(|e| { - if matches!(e, wasm_pkg_client::Error::NoRegistryForNamespace(_)) && registry.is_none() { - anyhow!("No default registry specified for wasm-pkg-loader. Create a default config, or set `registry` for package {package:?}") - } else { - e.into() - } - })?; - - releases.sort(); - - let release_version = releases - .iter() - .rev() - .find(|release| version.matches(&release.version) && !release.yanked) - .with_context(|| format!("No matching version found for {package} {version}",))?; - - let release = pkg_loader - .get_release(package, &release_version.version) - .await?; - - let digest = match &release.content_digest { - wasm_pkg_client::ContentDigest::Sha256 { hex } => format!("sha256:{hex}"), - }; - - let path = if let Ok(cached_path) = self.cache.wasm_file(&digest) { - cached_path - } else { - let mut stm = pkg_loader.stream_content(package, &release).await?; - - self.cache.ensure_dirs().await?; - let dest = self.cache.wasm_path(&digest); - - let mut file = tokio::fs::File::create(&dest).await?; - while let Some(block) = stm.next().await { - let bytes = block.context("Failed to get content from registry")?; - file.write_all(&bytes) - .await - .context("Failed to save registry content to cache")?; - } - - dest - }; - - file_content_ref(path) - } - // Copy content(s) from the given `mount` async fn copy_file_mounts( &self, @@ -789,6 +632,217 @@ fn locked_trigger(trigger_type: String, trigger: v2::Trigger) -> Result, +} + +impl WasmLoader { + /// Create a new instance of WasmLoader. + pub async fn new( + app_root: PathBuf, + cache_root: Option, + file_loading_permits: Option>, + ) -> Result { + let file_loading_permits = file_loading_permits.unwrap_or_else(|| { + std::sync::Arc::new(Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY)) + }); + Ok(Self { + app_root, + cache: Cache::new(cache_root).await?, + file_loading_permits, + }) + } + + /// Load a Wasm source from the given ComponentSource and return a path + /// to a file location from where it can be read. + pub async fn load_component_source( + &self, + component_id: &str, + source: &v2::ComponentSource, + ) -> Result { + let content = match source { + v2::ComponentSource::Local(path) => self.app_root.join(path), + v2::ComponentSource::Remote { url, digest } => { + self.load_http_source(url, digest).await? + } + v2::ComponentSource::Registry { + registry, + package, + version, + } => { + let version = semver::Version::parse(version).with_context(|| format!("Component {component_id} specifies an invalid semantic version ({version:?}) for its package version"))?; + let version_req = format!("={version}").parse().expect("version"); + + self.load_registry_source(registry.as_ref(), package, &version_req) + .await? + } + }; + Ok(content) + } + + // Load a Wasm source from the given HTTP ContentRef source URL and + // return a ContentRef an absolute path to the local copy. + async fn load_http_source(&self, url: &str, digest: &str) -> Result { + ensure!( + digest.starts_with("sha256:"), + "invalid `digest` {digest:?}; must start with 'sha256:'" + ); + let path = if let Ok(cached_path) = self.cache.wasm_file(digest) { + cached_path + } else { + let _loading_permit = self.file_loading_permits.acquire().await?; + + self.cache.ensure_dirs().await?; + let dest = self.cache.wasm_path(digest); + verified_download( + url, + digest, + &dest, + crate::http::DestinationConvention::ContentIndexed, + ) + .await + .with_context(|| format!("Error fetching source URL {url:?}"))?; + dest + }; + Ok(path) + } + + async fn load_registry_source( + &self, + registry: Option<&wasm_pkg_client::Registry>, + package: &wasm_pkg_client::PackageRef, + version: &semver::VersionReq, + ) -> Result { + let mut client_config = wasm_pkg_client::Config::global_defaults().await?; + + if let Some(registry) = registry.cloned() { + let mapping = wasm_pkg_client::RegistryMapping::Registry(registry); + client_config.set_package_registry_override(package.clone(), mapping); + } + let pkg_loader = wasm_pkg_client::Client::new(client_config); + + let mut releases = pkg_loader.list_all_versions(package).await.map_err(|e| { + if matches!(e, wasm_pkg_client::Error::NoRegistryForNamespace(_)) && registry.is_none() { + anyhow!("No default registry specified for wasm-pkg-loader. Create a default config, or set `registry` for package {package:?}") + } else { + e.into() + } + })?; + + releases.sort(); + + let release_version = releases + .iter() + .rev() + .find(|release| version.matches(&release.version) && !release.yanked) + .with_context(|| format!("No matching version found for {package} {version}",))?; + + let release = pkg_loader + .get_release(package, &release_version.version) + .await?; + + let digest = match &release.content_digest { + wasm_pkg_client::ContentDigest::Sha256 { hex } => format!("sha256:{hex}"), + }; + + let path = if let Ok(cached_path) = self.cache.wasm_file(&digest) { + cached_path + } else { + let mut stm = pkg_loader.stream_content(package, &release).await?; + + self.cache.ensure_dirs().await?; + let dest = self.cache.wasm_path(&digest); + + let mut file = tokio::fs::File::create(&dest).await?; + while let Some(block) = stm.next().await { + let bytes = block.context("Failed to get content from registry")?; + file.write_all(&bytes) + .await + .context("Failed to save registry content to cache")?; + } + + dest + }; + + Ok(path) + } + + /// Loads a dependency + pub async fn load_component_dependency( + &self, + dependency_name: &DependencyName, + dependency: &v2::ComponentDependency, + ) -> Result<(PathBuf, Option)> { + match dependency.clone() { + v2::ComponentDependency::Version(version) => { + let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; + + // This `unwrap()` should be OK because we've already validated + // this form of dependency requires a package name, i.e. the + // dependency name is not a kebab id. + let package = dependency_name.package().unwrap(); + + let content = self.load_registry_source(None, package, &version).await?; + Ok((content, None)) + } + v2::ComponentDependency::Package { + version, + registry, + package, + export, + } => { + let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; + + let package = match package { + Some(package) => { + package.parse().with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid package name ({package:?})"))? + } + None => { + // This `unwrap()` should be OK because we've already validated + // this form of dependency requires a package name, i.e. the + // dependency name is not a kebab id. + dependency_name + .package() + .cloned() + .unwrap() + } + }; + + let registry = match registry { + Some(registry) => { + registry + .parse() + .map(Some) + .with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid registry name ({registry:?})"))? + } + None => None, + }; + + let content = self + .load_registry_source(registry.as_ref(), &package, &version) + .await?; + Ok((content, export)) + } + v2::ComponentDependency::Local { path, export } => { + let content = self.app_root.join(path); + Ok((content, export)) + } + v2::ComponentDependency::HTTP { + url, + digest, + export, + } => { + let content = self.load_http_source(&url, &digest).await?; + Ok((content, export)) + } + } + } +} + fn looks_like_glob_pattern(s: impl AsRef) -> bool { let s = s.as_ref(); glob::Pattern::escape(s) != s diff --git a/crates/manifest/src/compat.rs b/crates/manifest/src/compat.rs index b16dd0bf63..4cca17cd3a 100644 --- a/crates/manifest/src/compat.rs +++ b/crates/manifest/src/compat.rs @@ -20,6 +20,7 @@ pub fn v1_to_v2_app(manifest: v1::AppManifestV1) -> Result, + /// `targets = ["spin-2.5", "fermyon-cloud", "spinkube-0.4"]` + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub targets: Vec, /// `[application.triggers.]` #[serde(rename = "trigger", default, skip_serializing_if = "Map::is_empty")] #[schemars(schema_with = "json_schema::map_of_toml_tables")] @@ -320,7 +323,7 @@ impl ComponentDependencies { } } - anyhow::bail!("{this:?} dependency conflicts with {other:?}") + Err(anyhow!("{this:?} dependency conflicts with {other:?}")) } /// Normalize version to perform a compatibility check against another version. @@ -351,6 +354,28 @@ impl ComponentDependencies { } } +/// Identifies a deployment target. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(untagged, deny_unknown_fields)] +pub enum TargetEnvironmentRef { + /// Environment definition doc reference e.g. `spin-up@3.2`. This is looked up + /// in the default environment catalogue (registry). + DefaultRegistry(String), + /// An environment definition doc in a registry other than the default + Registry { + /// Registry hosting the environment package e.g. `spinframewoek.dev`. + registry: String, + /// Environment definition doc name e.g. `my-spin-env@1.2`. + id: String, + }, + /// A local environment document file. This is expected to contain a serialised + /// EnvironmentDefinition in TOML format. + File { + /// The file path of the document. + path: PathBuf, + }, +} + mod kebab_or_snake_case { use serde::{Deserialize, Serialize}; pub use spin_serde::{KebabId, SnakeId}; diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index 9d249ccfb5..a648799ded 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -4198,6 +4198,32 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "indexmap 2.9.0", + "schemars_derive", + "semver", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.100", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4266,6 +4292,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "serde_json" version = "1.0.140" @@ -4806,6 +4843,7 @@ version = "3.3.0-pre0" dependencies = [ "anyhow", "indexmap 2.9.0", + "schemars", "semver", "serde", "spin-serde", @@ -4880,6 +4918,7 @@ version = "3.3.0-pre0" dependencies = [ "anyhow", "base64 0.22.1", + "schemars", "semver", "serde", "wasm-pkg-common", diff --git a/src/commands/build.rs b/src/commands/build.rs index 16270640e9..042dfbf337 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -29,6 +29,16 @@ pub struct BuildCommand { #[clap(short = 'c', long, multiple = true)] pub component_id: Vec, + /// By default, if the application manifest specifies one or more deployment targets, Spin + /// checks that all components are compatible with those deployment targets. Specify + /// this option to bypass those target checks. + #[clap( + long = "skip-target-checks", + alias = "skip-target-check", + takes_value = false + )] + skip_target_checks: bool, + /// Run the application after building. #[clap(name = BUILD_UP_OPT, short = 'u', long = "up")] pub up: bool, @@ -43,7 +53,13 @@ impl BuildCommand { spin_common::paths::find_manifest_file_path(self.app_source.as_ref())?; notify_if_nondefault_rel(&manifest_file, distance); - spin_build::build(&manifest_file, &self.component_id).await?; + spin_build::build( + &manifest_file, + &self.component_id, + self.target_checking(), + None, + ) + .await?; if self.up { let mut cmd = UpCommand::parse_from( @@ -59,4 +75,12 @@ impl BuildCommand { Ok(()) } } + + fn target_checking(&self) -> spin_build::TargetChecking { + if self.skip_target_checks { + spin_build::TargetChecking::Skip + } else { + spin_build::TargetChecking::Check + } + } } diff --git a/src/commands/registry.rs b/src/commands/registry.rs index 39a9cc4800..8370f4b1bb 100644 --- a/src/commands/registry.rs +++ b/src/commands/registry.rs @@ -58,7 +58,7 @@ pub struct Push { #[clap(long = "compose", default_value_t = true)] pub compose: bool, - /// Specifies to perform `spin build` before pushing the application. + /// Specifies to perform `spin build` (with the default options) before pushing the application. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] pub build: bool, @@ -84,7 +84,7 @@ impl Push { notify_if_nondefault_rel(&app_file, distance); if self.build { - spin_build::build(&app_file, &[]).await?; + spin_build::build_default(&app_file, self.cache_dir.clone()).await?; } let annotations = if self.annotations.is_empty() { diff --git a/src/commands/up.rs b/src/commands/up.rs index 9e11fb58cf..d0d565dcf0 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -108,7 +108,7 @@ pub struct UpCommand { #[clap(long, takes_value = false)] pub direct_mounts: bool, - /// For local apps, specifies to perform `spin build` before running the application. + /// For local apps, specifies to perform `spin build` (with the default options) before running the application. /// /// This is ignored on remote applications, as they are already built. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] @@ -191,7 +191,7 @@ impl UpCommand { } if self.build { - app_source.build().await?; + app_source.build(&self.cache_dir).await?; } let mut locked_app = self .load_resolved_app_source(resolved_app_source, &working_dir) diff --git a/src/commands/up/app_source.rs b/src/commands/up/app_source.rs index 29088f0ddd..dc5e0a179a 100644 --- a/src/commands/up/app_source.rs +++ b/src/commands/up/app_source.rs @@ -56,9 +56,9 @@ impl AppSource { } } - pub async fn build(&self) -> anyhow::Result<()> { + pub async fn build(&self, cache_root: &Option) -> anyhow::Result<()> { match self { - Self::File(path) => spin_build::build(path, &[]).await, + Self::File(path) => spin_build::build_default(path, cache_root.clone()).await, _ => Ok(()), } }