Introduce configuration version 2 (#208)

### What

This PR adds a new version (`"2"`) of the deployment configuration data
format.

This version of the configuration is capable of expressing array types
in collections and arguments.

Since this is the first time a new version is introduced there are a lot
of changes the only purpose of which is to distinguish between versions.

Only the infrastructure-related shell of the connector is aware of
different versions of deployment configurations existing. The core of
the connector only works with a single internal version.

This PR is also the one to introduce tests of array types. In hindsight
this ought to have been possible in the previous PR that introduced the
internal types and transformations (#191).

Note that there is not yet any automated way to upgrade a configuration
to a newer version, but this will be introduced shortly.

This PR also adds a changelog entry.

### How

The file `version2.rs` is a duplicate of `version1.rs`, which has been
adapted to use the new data types (incidentally these are just the ones
of the internal model).

`configuration.sql` now exists as `version1.sql` and `version2.sql`
respectively, since these have different capabilities. This is because
`version2.sql` introduces the ability to introspect array types.

`configuration.rs` now exposes `RawConfiguration` and `Configuration`
types which are enums of all the supported versions (currently 1 and
"2").
One big wart on the implementation is that serde and schemars are unable
to derive trait implementations for these types correctly, since they
only support strings as enum tags, and we used a number literal for
version 1.

Once we drop support of version 1 completely we can remove the manually
implemented instances.

The various `Connector` trait implementations now explicitly work on the
internal representation of a configuration, `RuntimeConfiguration`.

---------

Co-authored-by: Daniel Harvey <[email protected]>

Author: plcplc

.github/test-configuration.json

@@ -15,7 +15,7 @@
   },
   "ndc-postgres-12": {
     "feature": "postgres",
-    "filter": "not test(configure_initial_configuration_is_unchanged) and not test(configure_is_idempotent)",
+    "filter": "not test(postgres_current_only)",
     "services": "postgres",
     "env": {
       "POSTGRESQL_CONNECTION_STRING": "postgresql://postgres:password@localhost:64002",
@@ -24,7 +24,7 @@
   },
   "ndc-postgres-13": {
     "feature": "postgres",
-    "filter": "not test(configure_initial_configuration_is_unchanged) and not test(configure_is_idempotent)",
+    "filter": "not test(postgres_current_only)",
     "services": "postgres",
     "env": {
       "POSTGRESQL_CONNECTION_STRING": "postgresql://postgres:password@localhost:64002",
@@ -33,7 +33,7 @@
   },
   "ndc-postgres-14": {
     "feature": "postgres",
-    "filter": "not test(configure_initial_configuration_is_unchanged) and not test(configure_is_idempotent)",
+    "filter": "not test(postgres_current_only)",
     "services": "postgres",
     "env": {
       "POSTGRESQL_CONNECTION_STRING": "postgresql://postgres:password@localhost:64002",
@@ -42,7 +42,7 @@
   },
   "ndc-postgres-15": {
     "feature": "postgres",
-    "filter": "not test(configure_initial_configuration_is_unchanged) and not test(configure_is_idempotent)",
+    "filter": "not test(postgres_current_only)",
     "services": "postgres",
     "env": {
       "POSTGRESQL_CONNECTION_STRING": "postgresql://postgres:password@localhost:64002",

.github/workflows/cargo-test.yaml

@@ -127,9 +127,13 @@ jobs:
           AURORA_CONNECTION_STRING: ${{ secrets.AURORA_CONNECTION_STRING }}
         run: |
           # take connection string from env, create deployment file with it
-          cat static/aurora/chinook-deployment-template.json \
+          cat static/aurora/v1-chinook-deployment-template.json \
             | jq '.connectionUri={"uri":{"value":(env | .AURORA_CONNECTION_STRING)}}' \
-           > static/aurora/chinook-deployment.json
+           > static/aurora/v1-chinook-deployment.json
+
+          cat static/aurora/v2-chinook-deployment-template.json \
+            | jq '.connectionUri={"uri":{"value":(env | .AURORA_CONNECTION_STRING)}}' \
+           > static/aurora/v2-chinook-deployment.json
 
       - name: install tools
         run: |

changelog.md

@@ -4,6 +4,7 @@
 
 ### Added
 
+- Introduce version 2 of connector deployment configuration. ([#208](https://github.com/hasura/ndc-postgres/pull/208))
 - Support array types ([#191](https://github.com/hasura/ndc-postgres/pull/191), ...)
 - Support Native Query Mutations ([#189](https://github.com/hasura/ndc-postgres/pull/189), [#198](https://github.com/hasura/ndc-postgres/pull/198))
 

crates/connectors/ndc-postgres/src/configuration.rs

@@ -1,17 +1,70 @@
 //! Configuration for the connector.
 
-mod version1;
-
-use std::collections::BTreeSet;
+mod custom_trait_implementations;
+pub mod version1;
+pub mod version2;
 
+use custom_trait_implementations::RawConfigurationCompat;
+use ndc_sdk::connector;
 use query_engine_metadata::metadata;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+pub use version2::{occurring_scalar_types, ConnectionUri, PoolSettings, ResolvedSecret};
 
-pub use version1::{
-    configure, metadata_to_current, validate_raw_configuration, Configuration, ConnectionUri,
-    PoolSettings, RawConfiguration, ResolvedSecret,
-};
+/// Initial configuration, just enough to connect to a database and elaborate a full
+/// 'Configuration'.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(tag = "version")]
+#[serde(try_from = "RawConfigurationCompat")]
+#[serde(into = "RawConfigurationCompat")]
+// NOTE: Any changes to this data type will need follow-up changes to the
+// 'custom_trait_implementations' module.
+pub enum RawConfiguration {
+    // Until https://github.com/serde-rs/serde/pull/2525 is merged enum tags have to be strings.
+    #[serde(rename = "1")]
+    Version1(version1::RawConfiguration),
+    #[serde(rename = "2")]
+    Version2(version2::RawConfiguration),
+}
+
+impl RawConfiguration {
+    pub fn empty() -> Self {
+        RawConfiguration::Version2(version2::RawConfiguration::empty())
+    }
+}
 
-pub const CURRENT_VERSION: u32 = 1;
+/// User configuration, elaborated from a 'RawConfiguration'.
+#[derive(Debug, Deserialize, Serialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Configuration {
+    pub config: RawConfiguration,
+}
+
+pub async fn configure(
+    args: RawConfiguration,
+) -> Result<RawConfiguration, connector::UpdateConfigurationError> {
+    match args {
+        RawConfiguration::Version1(v1) => {
+            Ok(RawConfiguration::Version1(version1::configure(v1).await?))
+        }
+        RawConfiguration::Version2(v2) => {
+            Ok(RawConfiguration::Version2(version2::configure(v2).await?))
+        }
+    }
+}
+pub async fn validate_raw_configuration(
+    config: RawConfiguration,
+) -> Result<Configuration, connector::ValidateError> {
+    match config {
+        RawConfiguration::Version1(v1) => Ok(Configuration {
+            config: RawConfiguration::Version1(version1::validate_raw_configuration(v1).await?),
+        }),
+        RawConfiguration::Version2(v2) => Ok(Configuration {
+            config: RawConfiguration::Version2(version2::validate_raw_configuration(v2).await?),
+        }),
+    }
+}
 
 /// A configuration type, tailored to the needs of the query/mutation/explain methods (i.e., those
 /// not to do with configuration management).
@@ -25,54 +78,41 @@ pub const CURRENT_VERSION: u32 = 1;
 #[derive(Debug)]
 pub struct RuntimeConfiguration {
     pub metadata: metadata::Metadata,
+    pub pool_settings: version1::PoolSettings,
+    pub connection_uri: String,
 }
 
-impl<'a> version1::Configuration {
-    /// Apply the common interpretations on the Configuration API type into an RuntimeConfiguration.
-    pub fn as_runtime_configuration(self: &'a Configuration) -> RuntimeConfiguration {
-        RuntimeConfiguration {
-            metadata: metadata_to_current(&self.config.metadata),
-        }
+/// Apply the common interpretations on the Configuration API type into an RuntimeConfiguration.
+pub fn as_runtime_configuration(config: &Configuration) -> RuntimeConfiguration {
+    match &config.config {
+        RawConfiguration::Version1(v1_config) => RuntimeConfiguration {
+            metadata: version1::metadata_to_current(&v1_config.metadata),
+            pool_settings: v1_config.pool_settings.clone(),
+            connection_uri: match &v1_config.connection_uri {
+                ConnectionUri::Uri(ResolvedSecret(uri)) => uri.clone(),
+            },
+        },
+        RawConfiguration::Version2(v2_config) => RuntimeConfiguration {
+            metadata: v2_config.metadata.clone(),
+            pool_settings: v2_config.pool_settings.clone(),
+            connection_uri: match &v2_config.connection_uri {
+                ConnectionUri::Uri(ResolvedSecret(uri)) => uri.clone(),
+            },
+        },
     }
 }
 
-/// Collect all the types that can occur in the metadata. This is a bit circumstantial. A better
-/// approach is likely to record scalar type names directly in the metadata via configuration.sql.
-pub fn occurring_scalar_types(
-    tables: &metadata::TablesInfo,
-    native_queries: &metadata::NativeQueries,
-) -> BTreeSet<metadata::ScalarType> {
-    let tables_column_types = tables.0.values().flat_map(|v| {
-        v.columns
-            .values()
-            .map(|c| c.r#type.clone())
-            .filter_map(some_scalar_type)
-    });
-
-    let native_queries_column_types = native_queries.0.values().flat_map(|v| {
-        v.columns
-            .values()
-            .map(|c| c.r#type.clone())
-            .filter_map(some_scalar_type)
-    });
-
-    let native_queries_arguments_types = native_queries.0.values().flat_map(|v| {
-        v.arguments
-            .values()
-            .map(|c| c.r#type.clone())
-            .filter_map(some_scalar_type)
-    });
-
-    tables_column_types
-        .chain(native_queries_column_types)
-        .chain(native_queries_arguments_types)
-        .collect::<BTreeSet<metadata::ScalarType>>()
-}
+// for tests
 
-/// Filter predicate that only keeps scalar types.
-fn some_scalar_type(typ: metadata::Type) -> Option<metadata::ScalarType> {
-    match typ {
-        metadata::Type::ArrayType(_) => None,
-        metadata::Type::ScalarType(t) => Some(t),
+pub fn set_connection_uri(config: RawConfiguration, connection_uri: String) -> RawConfiguration {
+    match config {
+        RawConfiguration::Version1(v1) => RawConfiguration::Version1(version1::RawConfiguration {
+            connection_uri: ConnectionUri::Uri(ResolvedSecret(connection_uri)),
+            ..v1
+        }),
+        RawConfiguration::Version2(v2) => RawConfiguration::Version2(version2::RawConfiguration {
+            connection_uri: ConnectionUri::Uri(ResolvedSecret(connection_uri)),
+            ..v2
+        }),
     }
 }