start establishing the concept of "artifacts"

Start moving towards the cloud-assembly specification where an output
of a CDK program is a bunch of artifacts and those are processed by
the toolkit.

Related #956
Related #233
Related #1119

Author: Elad Ben-Israel

design/cloud-assembly.md

@@ -1,7 +1,6 @@
 import cxapi = require('@aws-cdk/cx-api');
-import { Stack } from './cloudformation/stack';
-import { IConstruct, Root } from './core/construct';
-import { InMemorySynthesisSession, ISynthesisSession, SynthesisSession } from './synthesis';
+import { Root } from './core/construct';
+import { FileSystemStore, InMemoryStore, ISynthesisSession, SynthesisSession } from './synthesis';
 
 /**
  * Represents a CDK program.
@@ -18,22 +17,6 @@ export class App extends Root {
     this.loadContext();
   }
 
-  private get stacks() {
-    const out: { [name: string]: Stack } = { };
-    collectStacks(this);
-    return out;
-
-    function collectStacks(c: IConstruct) {
-      for (const child of c.node.children) {
-        if (Stack.isStack(child)) {
-          out[child.node.id] = child; // TODO: this should probably be changed to uniqueId
-        }
-
-        collectStacks(child);
-      }
-    }
-  }
-
   /**
    * Runs the program. Output is written to output directory as specified in the request.
    */
@@ -44,13 +27,14 @@ export class App extends Root {
     }
 
     const outdir = process.env[cxapi.OUTDIR_ENV];
+    let store;
     if (outdir) {
-      this._session = new SynthesisSession({ outdir });
+      store = new FileSystemStore({ outdir });
     } else {
-      this._session = new InMemorySynthesisSession();
+      store = new InMemoryStore();
     }
 
-    const session = this._session;
+    const session = this._session = new SynthesisSession(store);
 
     // the three holy phases of synthesis: prepare, validate and synthesize
 
@@ -67,18 +51,7 @@ export class App extends Root {
     // synthesize
     this.node.synthesizeTree(session);
 
-    // write the entrypoint/manifest of this app. It includes a *copy* of the
-    // synthesized stack output for backwards compatibility
-
-    const manifest: cxapi.SynthesizeResponse = {
-      version: cxapi.PROTO_RESPONSE_VERSION,
-      stacks: Object.values(this.stacks).map(s => this.readSynthesizedStack(session, s.artifactName)),
-      runtime: this.collectRuntimeInformation()
-    };
-
-    session.writeFile(cxapi.OUTFILE_NAME, JSON.stringify(manifest, undefined, 2));
-
-    // lock session - cannot emit more artifacts
+    // write session manifest and lock store
     session.finalize();
 
     return session;
@@ -90,9 +63,13 @@ export class App extends Root {
    * @deprecated This method is going to be deprecated in a future version of the CDK
    */
   public synthesizeStack(stackName: string): cxapi.SynthesizedStack {
-    const stack = this.getStack(stackName);
     const session = this.run();
-    return this.readSynthesizedStack(session, stack.artifactName);
+    const res = session.manifest.stacks.find(s => s.name === stackName);
+    if (!res) {
+      throw new Error(`Stack "${stackName}" not found`);
+    }
+
+    return res;
   }
 
   /**
@@ -107,46 +84,6 @@ export class App extends Root {
     return ret;
   }
 
-  private readSynthesizedStack(session: ISynthesisSession, artifactName: string) {
-    return JSON.parse(session.readFile(artifactName).toString());
-  }
-
-  private collectRuntimeInformation(): cxapi.AppRuntime {
-    const libraries: { [name: string]: string } = {};
-
-    for (const fileName of Object.keys(require.cache)) {
-      const pkg = findNpmPackage(fileName);
-      if (pkg && !pkg.private) {
-        libraries[pkg.name] = pkg.version;
-      }
-    }
-
-    // include only libraries that are in the @aws-cdk npm scope
-    for (const name of Object.keys(libraries)) {
-      if (!name.startsWith('@aws-cdk/')) {
-        delete libraries[name];
-      }
-    }
-
-    // add jsii runtime version
-    libraries['jsii-runtime'] = getJsiiAgentVersion();
-
-    return { libraries };
-  }
-
-  private getStack(stackname: string) {
-    if (stackname == null) {
-      throw new Error('Stack name must be defined');
-    }
-
-    const stack = this.stacks[stackname];
-
-    if (!stack) {
-      throw new Error(`Cannot find stack ${stackname}`);
-    }
-    return stack;
-  }
-
   private loadContext() {
     const contextJson = process.env[cxapi.CONTEXT_ENV];
     const context = !contextJson ? { } : JSON.parse(contextJson);
@@ -155,64 +92,3 @@ export class App extends Root {
     }
   }
 }
-
-/**
- * Determines which NPM module a given loaded javascript file is from.
- *
- * The only infromation that is available locally is a list of Javascript files,
- * and every source file is associated with a search path to resolve the further
- * ``require`` calls made from there, which includes its own directory on disk,
- * and parent directories - for example:
- *
- * [ '...repo/packages/aws-cdk-resources/lib/cfn/node_modules',
- *   '...repo/packages/aws-cdk-resources/lib/node_modules',
- *   '...repo/packages/aws-cdk-resources/node_modules',
- *   '...repo/packages/node_modules',
- *   // etc...
- * ]
- *
- * We are looking for ``package.json`` that is anywhere in the tree, except it's
- * in the parent directory, not in the ``node_modules`` directory. For this
- * reason, we strip the ``/node_modules`` suffix off each path and use regular
- * module resolution to obtain a reference to ``package.json``.
- *
- * @param fileName a javascript file name.
- * @returns the NPM module infos (aka ``package.json`` contents), or
- *      ``undefined`` if the lookup was unsuccessful.
- */
-function findNpmPackage(fileName: string): { name: string, version: string, private?: boolean } | undefined {
-  const mod = require.cache[fileName];
-  const paths = mod.paths.map(stripNodeModules);
-
-  try {
-    const packagePath = require.resolve('package.json', { paths });
-    return require(packagePath);
-  } catch (e) {
-    return undefined;
-  }
-
-  /**
-   * @param s a path.
-   * @returns ``s`` with any terminating ``/node_modules``
-   *      (or ``\\node_modules``) stripped off.)
-   */
-  function stripNodeModules(s: string): string {
-    if (s.endsWith('/node_modules') || s.endsWith('\\node_modules')) {
-      // /node_modules is 13 characters
-      return s.substr(0, s.length - 13);
-    }
-    return s;
-  }
-}
-
-function getJsiiAgentVersion() {
-  let jsiiAgent = process.env.JSII_AGENT;
-
-  // if JSII_AGENT is not specified, we will assume this is a node.js runtime
-  // and plug in our node.js version
-  if (!jsiiAgent) {
-    jsiiAgent = `node.js/${process.version}`;
-  }
-
-  return jsiiAgent;
-}

packages/@aws-cdk/cdk/lib/app.ts

@@ -78,11 +78,6 @@ export class Stack extends Construct {
    */
   public readonly name: string;
 
-  /**
-   * The name of the CDK artifact produced by this stack.
-   */
-  public readonly artifactName: string;
-
   /*
    * Used to determine if this construct is a stack.
    */
@@ -112,8 +107,6 @@ export class Stack extends Construct {
 
     this.logicalIds = new LogicalIDs(props && props.namingScheme ? props.namingScheme : new HashedAddressingScheme());
     this.name = this.node.id;
-
-    this.artifactName = `${this.node.uniqueId}.stack.json`;
   }
 
   /**
@@ -428,25 +421,21 @@ export class Stack extends Construct {
   protected synthesize(session: ISynthesisSession): void {
     const account = this.env.account || 'unknown-account';
     const region = this.env.region || 'unknown-region';
-
-    const environment: cxapi.Environment = {
-      name: `${account}/${region}`,
-      account,
-      region
-    };
-
     const missing = Object.keys(this.missingContext).length ? this.missingContext : undefined;
+    const template = `${this.node.id}.template.json`;
 
-    const output: cxapi.SynthesizedStack = {
-      name: this.node.id,
-      template: this.toCloudFormation(),
-      environment,
-      missing,
-      metadata: this.collectMetadata(),
-      dependsOn: noEmptyArray(this.dependencies().map(s => s.node.id)),
-    };
+    // write the CloudFormation template as a JSON file
+    session.store.writeJson(template, this.toCloudFormation());
 
-    session.writeFile(this.artifactName, JSON.stringify(output, undefined, 2));
+    // add an artifact that represents this stack
+    session.addArtifact(this.node.id, {
+      type: cxapi.ArtifactType.CloudFormationStack,
+      dependencies: noEmptyArray(this.dependencies().map(s => s.node.id)),
+      environment: `aws://${account}/${region}`,
+      metadata: this.collectMetadata(),
+      missing,
+      properties: { template }
+    });
   }
 
   /**

packages/@aws-cdk/cdk/lib/cloudformation/stack.ts

@@ -0,0 +1,85 @@
+import cxapi = require('@aws-cdk/cx-api');
+
+export function collectRuntimeInformation(): cxapi.AppRuntime {
+  const libraries: { [name: string]: string } = {};
+
+  for (const fileName of Object.keys(require.cache)) {
+    const pkg = findNpmPackage(fileName);
+    if (pkg && !pkg.private) {
+      libraries[pkg.name] = pkg.version;
+    }
+  }
+
+  // include only libraries that are in the @aws-cdk npm scope
+  for (const name of Object.keys(libraries)) {
+    if (!name.startsWith('@aws-cdk/')) {
+      delete libraries[name];
+    }
+  }
+
+  // add jsii runtime version
+  libraries['jsii-runtime'] = getJsiiAgentVersion();
+
+  return { libraries };
+}
+
+/**
+ * Determines which NPM module a given loaded javascript file is from.
+ *
+ * The only infromation that is available locally is a list of Javascript files,
+ * and every source file is associated with a search path to resolve the further
+ * ``require`` calls made from there, which includes its own directory on disk,
+ * and parent directories - for example:
+ *
+ * [ '...repo/packages/aws-cdk-resources/lib/cfn/node_modules',
+ *   '...repo/packages/aws-cdk-resources/lib/node_modules',
+ *   '...repo/packages/aws-cdk-resources/node_modules',
+ *   '...repo/packages/node_modules',
+ *   // etc...
+ * ]
+ *
+ * We are looking for ``package.json`` that is anywhere in the tree, except it's
+ * in the parent directory, not in the ``node_modules`` directory. For this
+ * reason, we strip the ``/node_modules`` suffix off each path and use regular
+ * module resolution to obtain a reference to ``package.json``.
+ *
+ * @param fileName a javascript file name.
+ * @returns the NPM module infos (aka ``package.json`` contents), or
+ *      ``undefined`` if the lookup was unsuccessful.
+ */
+function findNpmPackage(fileName: string): { name: string, version: string, private?: boolean } | undefined {
+  const mod = require.cache[fileName];
+  const paths = mod.paths.map(stripNodeModules);
+
+  try {
+    const packagePath = require.resolve('package.json', { paths });
+    return require(packagePath);
+  } catch (e) {
+    return undefined;
+  }
+
+  /**
+   * @param s a path.
+   * @returns ``s`` with any terminating ``/node_modules``
+   *      (or ``\\node_modules``) stripped off.)
+   */
+  function stripNodeModules(s: string): string {
+    if (s.endsWith('/node_modules') || s.endsWith('\\node_modules')) {
+      // /node_modules is 13 characters
+      return s.substr(0, s.length - 13);
+    }
+    return s;
+  }
+}
+
+function getJsiiAgentVersion() {
+  let jsiiAgent = process.env.JSII_AGENT;
+
+  // if JSII_AGENT is not specified, we will assume this is a node.js runtime
+  // and plug in our node.js version
+  if (!jsiiAgent) {
+    jsiiAgent = `node.js/${process.version}`;
+  }
+
+  return jsiiAgent;
+}

packages/@aws-cdk/cdk/lib/runtime-info.ts

@@ -1,10 +1,67 @@
+import cxapi = require('@aws-cdk/cx-api');
 import fs = require('fs');
 import os = require('os');
 import path = require('path');
+import { collectRuntimeInformation } from './runtime-info';
 
 export interface ISynthesisSession {
+  readonly store: ISessionStore;
+  readonly manifest: cxapi.AssemblyManifest;
+  addArtifact(id: string, droplet: cxapi.Artifact): void;
+  tryGetArtifact(id: string): cxapi.Artifact | undefined;
+}
+
+export class SynthesisSession implements ISynthesisSession {
+  private readonly artifacts: { [id: string]: cxapi.Artifact } = { };
+  private _manifest?: cxapi.AssemblyManifest;
+
+  constructor(public readonly store: ISessionStore) {
+
+  }
+
+  public get manifest() {
+    if (!this._manifest) {
+      throw new Error(`Cannot read assembly manifest before the session has been finalized`);
+    }
+
+    return this._manifest;
+  }
+
+  public addArtifact(id: string, artifact: cxapi.Artifact): void {
+    cxapi.validateArtifact(artifact);
+    this.store.writeFile(id, JSON.stringify(artifact, undefined, 2));
+    this.artifacts[id] = artifact;
+  }
+
+  public tryGetArtifact(id: string): cxapi.Artifact | undefined {
+    if (!this.store.exists(id)) {
+      return undefined;
+    }
+
+    return JSON.parse(this.store.readFile(id).toString());
+  }
+
+  public finalize(): cxapi.AssemblyManifest {
+    const manifest: cxapi.SynthesizeResponse = this._manifest = {
+      version: cxapi.PROTO_RESPONSE_VERSION,
+      artifacts: this.artifacts,
+      runtime: collectRuntimeInformation(),
+
+      // for backwards compatbility
+      stacks: renderLegacyStacks(this.artifacts, this.store),
+    };
+
+    // write the manifest (under both legacy and new name)
+    this.store.writeFile(cxapi.OUTFILE_NAME, JSON.stringify(manifest, undefined, 2));
+    this.store.writeFile(cxapi.MANIFEST_FILE, JSON.stringify(manifest, undefined, 2));
+
+    return manifest;
+  }
+}
+
+export interface ISessionStore {
   /**
-   * Creates a directory under the session directory and returns it's full path.
+   * Creates a directory and returns it's full path.
    * @param directoryName The name of the directory to create.
    * @throws if a directory by that name already exists in the session or if the session has already been finalized.
    */
@@ -18,38 +75,56 @@ export interface ISynthesisSession {
   readdir(directoryName: string): string[];
 
   /**
-   * Writes a file into the synthesis session directory.
+   * Writes a file into the store.
    * @param artifactName The name of the file.
    * @param data The contents of the file.
    */
   writeFile(artifactName: string, data: any): void;
 
   /**
-   * Reads a file from the synthesis session directory.
+   * Writes a formatted JSON output file to the store
+   * @param artifactName the name of the artifact
+   * @param json the JSON object
+   */
+  writeJson(artifactName: string, json: any): void;
+
+  /**
+   * Reads a file from the store.
    * @param fileName The name of the file.
    * @throws if the file is not found
    */
   readFile(fileName: string): any;
 
   /**
-   * @returns true if the file `fileName` exists in the session directory.
+   * Reads a JSON object from the store.
+   */
+  readJson(fileName: string): any;
+
+  /**
+   * @returns true if the file `fileName` exists in the store.
    * @param name The name of the file or directory to look up.
    */
   exists(name: string): boolean;
 
   /**
-   * List all artifacts that were emitted to the session.
+   * List all top-level files that were emitted to the store.
    */
   list(): string[];
 
   /**
-   * Finalizes the session. After this is called, the session will be locked for
-   * writing.
+   * Do not allow further writes into the store.
    */
   finalize(): void;
 }
 
 export interface SynthesisSessionOptions {
+  /**
+   * Where to store the
+   */
+  store: ISessionStore;
+}
+
+export interface FileSystemStoreOptions {
   /**
    * The output directory for synthesis artifacts
    */
@@ -59,11 +134,11 @@ export interface SynthesisSessionOptions {
 /**
  * Can be used to prepare and emit synthesis artifacts into an output directory.
  */
-export class SynthesisSession implements ISynthesisSession {
+export class FileSystemStore implements ISessionStore {
   private readonly outdir: string;
   private locked = false;
 
-  constructor(options: SynthesisSessionOptions) {
+  constructor(options: FileSystemStoreOptions) {
     this.outdir = options.outdir;
     return;
   }
@@ -75,6 +150,10 @@ export class SynthesisSession implements ISynthesisSession {
     fs.writeFileSync(p, data);
   }
 
+  public writeJson(fileName: string, json: any) {
+    this.writeFile(fileName, JSON.stringify(json, undefined, 2));
+  }
+
   public readFile(fileName: string): any {
     const p = this.pathForArtifact(fileName);
     if (!fs.existsSync(p)) {
@@ -84,6 +163,10 @@ export class SynthesisSession implements ISynthesisSession {
     return fs.readFileSync(p);
   }
 
+  public readJson(fileName: string): any {
+    return JSON.parse(this.readFile(fileName).toString());
+  }
+
   public exists(name: string): boolean {
     const p = this.pathForArtifact(name);
     return fs.existsSync(p);
@@ -127,7 +210,7 @@ export class SynthesisSession implements ISynthesisSession {
   }
 }
 
-export class InMemorySynthesisSession implements ISynthesisSession {
+export class InMemoryStore implements ISessionStore {
   private files: { [fileName: string]: any } = { };
   private dirs: { [dirName: string]: string } = { }; // value is path to a temporary directory
 
@@ -138,13 +221,21 @@ export class InMemorySynthesisSession implements ISynthesisSession {
     this.files[fileName] = data;
   }
 
+  public writeJson(fileName: string, json: any): void {
+    this.writeFile(fileName, JSON.stringify(json, undefined, 2));
+  }
+
   public readFile(fileName: string) {
     if (!(fileName in this.files)) {
       throw new Error(`${fileName} not found`);
     }
     return this.files[fileName];
   }
 
+  public readJson(fileName: string): any {
+    return JSON.parse(this.readFile(fileName).toString());
+  }
+
   public exists(name: string) {
     return name in this.files || name in this.dirs;
   }
@@ -182,4 +273,43 @@ export class InMemorySynthesisSession implements ISynthesisSession {
       throw new Error('Session has already been finalized');
     }
   }
+}
+
+function renderLegacyStacks(artifacts: { [id: string]: cxapi.Artifact }, store: ISessionStore) {
+  // special case for backwards compat. build a list of stacks for the manifest
+  const stacks = new Array<cxapi.SynthesizedStack>();
+
+  for (const [ id, artifact ] of Object.entries(artifacts)) {
+    if (artifact.type === cxapi.ArtifactType.CloudFormationStack) {
+      const templateFile = (artifact.properties || {}).template;
+      if (!templateFile) {
+        throw new Error(`Invalid cloudformation artifact. Missing "template" property`);
+      }
+      const template = store.readJson(templateFile);
+
+      const match = cxapi.AWS_ENV_REGEX.exec(artifact.environment);
+      if (!match) {
+        throw new Error(`"environment" must match regex: ${cxapi.AWS_ENV_REGEX}`);
+      }
+
+      const synthStack: cxapi.SynthesizedStack = {
+        name: id,
+        environment: { name: artifact.environment.substr('aws://'.length), account: match[1], region: match[2] },
+        template,
+        metadata: artifact.metadata || {},
+      };
+
+      if (artifact.dependencies && artifact.dependencies.length > 0) {
+        synthStack.dependsOn = artifact.dependencies;
+      }
+
+      if (artifact.missing) {
+        synthStack.missing = artifact.missing;
+      }
+
+      stacks.push(synthStack);
+    }
+  }
+
+  return stacks;
 }

packages/@aws-cdk/cdk/lib/synthesis.ts

@@ -16,7 +16,7 @@ function withApp(context: { [key: string]: any } | undefined, block: (app: App)
 
   const session = app.run();
 
-  return JSON.parse(session.readFile(cxapi.OUTFILE_NAME));
+  return session.manifest;
 }
 
 function synth(context?: { [key: string]: any }): cxapi.SynthesizeResponse {
@@ -59,6 +59,7 @@ export = {
     // clean up metadata so assertion will be sane
     response.stacks.forEach(s => delete s.metadata);
     delete response.runtime;
+    delete response.artifacts;
 
     test.deepEqual(response, {
       version: '0.19.0',
@@ -90,13 +91,7 @@ export = {
     const stack = new Stack(prog, 'MyStack');
     new Resource(stack, 'MyResource', { type: 'MyResourceType' });
 
-    let throws;
-    try {
-      prog.synthesizeStacks(['foo']);
-    } catch (e) {
-      throws = e.message;
-    }
-    test.ok(throws.indexOf('Cannot find stack foo') !== -1);
+    test.throws(() => prog.synthesizeStacks(['foo']), /foo/);
 
     test.deepEqual(prog.synthesizeStack('MyStack').template,
       { Resources: { MyResource: { Type: 'MyResourceType' } } });
@@ -264,7 +259,7 @@ export = {
       new Resource(stack, 'MyResource', { type: 'Resource::Type' });
     });
 
-    const libs = response.runtime.libraries;
+    const libs = (response.runtime && response.runtime.libraries) || { };
 
     const version = require('../package.json').version;
     test.deepEqual(libs['@aws-cdk/cdk'], version);
@@ -281,7 +276,7 @@ export = {
       new Resource(stack, 'MyResource', { type: 'Resource::Type' });
     });
 
-    const libs = response.runtime.libraries;
+    const libs = (response.runtime && response.runtime.libraries) || { };
     test.deepEqual(libs['jsii-runtime'], `Java/1.2.3.4`);
 
     delete process.env.JSII_AGENT;
@@ -294,7 +289,7 @@ export = {
       new Resource(stack, 'MyResource', { type: 'Resource::Type' });
     });
 
-    const libs = response.runtime.libraries;
+    const libs = (response.runtime && response.runtime.libraries) || { };
 
     const version = require('../package.json').version;
     test.deepEqual(libs, {

packages/@aws-cdk/cdk/test/test.app.ts

@@ -4,39 +4,12 @@ import { Test } from 'nodeunit';
 import os = require('os');
 import path = require('path');
 import cdk = require('../lib');
-import { InMemorySynthesisSession, SynthesisSession } from '../lib';
+import { FileSystemStore, InMemoryStore, SynthesisSession } from '../lib';
 
-const sessionTestMatix: any = {};
+const storeTestMatrix: any = {};
 
 export = {
-  'constructs that implement "synthesize" can emit artifacts during synthesis'(test: Test) {
-    // GIVEN
-    const app = new cdk.App();
-    new Synthesizer1(app, 'synthe1');
-    const s2 = new Synthesizer2(app, 'synthe2');
-    new Synthesizer3(s2, 'synthe3');
-
-    // WHEN
-    const session = app.run();
-
-    // THEN
-    test.deepEqual(session.readFile('s1.txt'), 'hello, s1');
-    test.deepEqual(session.readFile('s2.txt'), 'hello, s2');
-
-    test.deepEqual(session.list(), [
-      'cdk.out',
-      's1.txt',
-      's2.txt',
-      'synthe2Group0512C945A.txt',
-      'synthe2Group181E95665.txt',
-      'synthe2Group20BD1A3CD.txt',
-      'synthe2synthe30CE80559.txt'
-    ]);
-
-    test.done();
-  },
-
-  'cdk.out contains all synthesized stacks'(test: Test) {
+  'backwards compatibility: cdk.out contains all synthesized stacks'(test: Test) {
     // GIVEN
     const app = new cdk.App();
     const stack1 = new cdk.Stack(app, 'stack1');
@@ -47,7 +20,7 @@ export = {
 
     // WHEN
     const session = app.run();
-    const manifest: cxapi.SynthesizeResponse = JSON.parse(session.readFile(cxapi.OUTFILE_NAME).toString());
+    const manifest = session.manifest;
 
     // THEN
     const t1 = manifest.stacks.find(s => s.name === 'stack1')!.template;
@@ -67,96 +40,184 @@ export = {
     test.done();
   },
 
-  'session': sessionTestMatix
+  'store': storeTestMatrix
 };
 
-const sessionTests = {
-  'writeFile()/readFile()'(test: Test, session: cdk.ISynthesisSession) {
+//
+// all these tests will be executed for each type of store
+//
+const storeTests = {
+  'writeFile()/readFile()'(test: Test, store: cdk.ISessionStore) {
     // WHEN
-    session.writeFile('bla.txt', 'hello');
-    session.writeFile('hey.txt', '1234');
+    store.writeFile('bla.txt', 'hello');
+    store.writeFile('hey.txt', '1234');
 
     // THEN
-    test.deepEqual(session.readFile('bla.txt').toString(), 'hello');
-    test.deepEqual(session.readFile('hey.txt').toString(), '1234');
-    test.throws(() => session.writeFile('bla.txt', 'override is forbidden'));
+    test.deepEqual(store.readFile('bla.txt').toString(), 'hello');
+    test.deepEqual(store.readFile('hey.txt').toString(), '1234');
+    test.throws(() => store.writeFile('bla.txt', 'override is forbidden'));
 
     // WHEN
-    session.finalize();
+    store.finalize();
 
     // THEN
-    test.throws(() => session.writeFile('another.txt', 'locked!'));
+    test.throws(() => store.writeFile('another.txt', 'locked!'));
     test.done();
   },
 
-  'exists() for files'(test: Test, session: cdk.ISynthesisSession) {
+  'exists() for files'(test: Test, store: cdk.ISessionStore) {
     // WHEN
-    session.writeFile('A.txt', 'aaa');
+    store.writeFile('A.txt', 'aaa');
 
     // THEN
-    test.ok(session.exists('A.txt'));
-    test.ok(!session.exists('B.txt'));
+    test.ok(store.exists('A.txt'));
+    test.ok(!store.exists('B.txt'));
     test.done();
   },
 
-  'mkdir'(test: Test, session: cdk.ISynthesisSession) {
+  'mkdir'(test: Test, store: cdk.ISessionStore) {
     // WHEN
-    const dir1 = session.mkdir('dir1');
-    const dir2 = session.mkdir('dir2');
+    const dir1 = store.mkdir('dir1');
+    const dir2 = store.mkdir('dir2');
 
     // THEN
     test.ok(fs.statSync(dir1).isDirectory());
     test.ok(fs.statSync(dir2).isDirectory());
-    test.throws(() => session.mkdir('dir1'));
+    test.throws(() => store.mkdir('dir1'));
 
     // WHEN
-    session.finalize();
-    test.throws(() => session.mkdir('dir3'));
+    store.finalize();
+    test.throws(() => store.mkdir('dir3'));
     test.done();
   },
 
-  'list'(test: Test, session: cdk.ISynthesisSession) {
+  'list'(test: Test, store: cdk.ISessionStore) {
     // WHEN
-    session.mkdir('dir1');
-    session.writeFile('file1.txt', 'boom1');
+    store.mkdir('dir1');
+    store.writeFile('file1.txt', 'boom1');
 
     // THEN
-    test.deepEqual(session.list(), ['dir1', 'file1.txt']);
+    test.deepEqual(store.list(), ['dir1', 'file1.txt']);
     test.done();
-  }
-};
+  },
 
-for (const [name, fn] of Object.entries(sessionTests)) {
-  const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'synthesis-tests'));
-  const fsSession = new SynthesisSession({ outdir });
-  const memorySession = new InMemorySynthesisSession();
-  sessionTestMatix[`SynthesisSession - ${name}`] = (test: Test) => fn(test, fsSession);
-  sessionTestMatix[`InMemorySession - ${name}`] = (test: Test) => fn(test, memorySession);
-}
+  'SynthesisSession'(test: Test, store: cdk.ISessionStore) {
+    // GIVEN
+    const session = new SynthesisSession(store);
+    const templateFile = 'foo.template.json';
 
-class Synthesizer1 extends cdk.Construct {
-  public synthesize(s: cdk.ISynthesisSession) {
-    s.writeFile('s1.txt', 'hello, s1');
-  }
-}
+    // WHEN
+    session.addArtifact('my-first-artifact', {
+      type: cxapi.ArtifactType.CloudFormationStack,
+      environment: 'aws://1222344/us-east-1',
+      dependencies: ['a', 'b'],
+      metadata: {
+        foo: { bar: 123 }
+      },
+      properties: {
+        template: templateFile,
+        prop1: 1234,
+        prop2: 555
+      },
+      missing: {
+        foo: {
+          provider: 'context-provider',
+          props: {
+            a: 'A',
+            b: 2
+          }
+        }
+      }
+    });
 
-class Synthesizer2 extends cdk.Construct {
-  constructor(scope: cdk.Construct, id: string) {
-    super(scope, id);
+    session.addArtifact('minimal-artifact', {
+      type: cxapi.ArtifactType.CloudFormationStack,
+      environment: 'aws://111/helo-world',
+      properties: {
+        template: templateFile
+      }
+    });
 
-    const group = new cdk.Construct(this, 'Group');
-    for (let i = 0; i < 3; ++i) {
-      new Synthesizer3(group, `${i}`);
-    }
-  }
+    session.store.writeJson(templateFile, {
+      Resources: {
+        MyTopic: {
+          Type: 'AWS::S3::Topic'
+        }
+      }
+    });
 
-  public synthesize(s: cdk.ISynthesisSession) {
-    s.writeFile('s2.txt', 'hello, s2');
-  }
-}
+    session.finalize();
 
-class Synthesizer3 extends cdk.Construct {
-  public synthesize(s: cdk.ISynthesisSession) {
-    s.writeFile(this.node.uniqueId + '.txt', 'hello, s3');
+    // THEN
+    delete session.manifest.stacks; // remove legacy
+    delete session.manifest.runtime; // deterministic tests
+
+    // verify the manifest looks right
+    test.deepEqual(session.manifest, {
+      version: cxapi.PROTO_RESPONSE_VERSION,
+      artifacts: {
+        'my-first-artifact': {
+          type: 'aws:cloudformation:stack',
+          environment: 'aws://1222344/us-east-1',
+          dependencies: ['a', 'b'],
+          metadata: { foo: { bar: 123 } },
+          properties: { template: 'foo.template.json', prop1: 1234, prop2: 555 },
+          missing: {
+            foo: { provider: 'context-provider', props: { a: 'A', b: 2 } }
+          }
+        },
+        'minimal-artifact': {
+          type: 'aws:cloudformation:stack',
+          environment: 'aws://111/helo-world',
+          properties: { template: 'foo.template.json' }
+        }
+      }
+    });
+
+    // verify we have a template file
+    test.deepEqual(session.store.readJson(templateFile), {
+      Resources: {
+        MyTopic: {
+          Type: 'AWS::S3::Topic'
+        }
+      }
+    });
+
+    test.done();
   }
+};
+
+for (const [name, fn] of Object.entries(storeTests)) {
+  const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'synthesis-tests'));
+  const fsStore = new FileSystemStore({ outdir });
+  const memoryStore = new InMemoryStore();
+  storeTestMatrix[`FileSystemStore - ${name}`] = (test: Test) => fn(test, fsStore);
+  storeTestMatrix[`InMemoryStore - ${name}`] = (test: Test) => fn(test, memoryStore);
 }
+
+// class Synthesizer1 extends cdk.Construct {
+//   public synthesize(s: cdk.ISynthesisSession) {
+//     s.writeFile('s1.txt', 'hello, s1');
+//   }
+// }
+
+// class Synthesizer2 extends cdk.Construct {
+//   constructor(scope: cdk.Construct, id: string) {
+//     super(scope, id);
+
+//     const group = new cdk.Construct(this, 'Group');
+//     for (let i = 0; i < 3; ++i) {
+//       new Synthesizer3(group, `${i}`);
+//     }
+//   }
+
+//   public synthesize(s: cdk.ISynthesisSession) {
+//     s.writeFile('s2.txt', 'hello, s2');
+//   }
+// }
+
+// class Synthesizer3 extends cdk.Construct {
+//   public synthesize(s: cdk.ISynthesisSession) {
+//     s.writeFile(this.node.uniqueId + '.txt', 'hello, s3');
+//   }
+// }

packages/@aws-cdk/cdk/test/test.synthesis.ts

@@ -0,0 +1,27 @@
+export const AWS_ENV_REGEX = /aws\:\/\/([0-9]+|unknown-account)\/([a-z\-0-9]+)/;
+
+export enum ArtifactType {
+  CloudFormationStack = 'aws:cloudformation:stack',
+  DockerImage = 'aws:docker',
+  File = 'aws:file'
+}
+
+export interface Artifact {
+  type: ArtifactType;
+  environment: string; // format: aws://account/region
+  properties?: { [name: string]: any };
+  metadata?: { [path: string]: any };
+  dependencies?: string[];
+  missing?: { [key: string]: any };
+
+  /**
+   * Build instructions for this artifact (for example, lambda-builders, zip directory, docker build, etc)
+   */
+  build?: any;
+}
+
+export function validateArtifact(artifcat: Artifact) {
+  if (!AWS_ENV_REGEX.test(artifcat.environment)) {
+    throw new Error(`Artifact "environment" must conform to ${AWS_ENV_REGEX}: ${artifcat.environment}`);
+  }
+}

packages/@aws-cdk/cx-api/lib/artifacts.ts

@@ -2,6 +2,7 @@
  * File with definitions for the interface between the Cloud Executable and the CDK toolkit.
  */
 
+import { Artifact } from './artifacts';
 import { Environment } from './environment';
 
 /**
@@ -22,7 +23,16 @@ import { Environment } from './environment';
  */
 export const PROTO_RESPONSE_VERSION = '0.19.0';
 
+/**
+ * @deprecated Use `MANIFEST_FILE`
+ */
 export const OUTFILE_NAME = 'cdk.out';
+
+/**
+ * The name of the root manifest file of the assembly.
+ */
+export const MANIFEST_FILE = "manifest.json";
+
 export const OUTDIR_ENV = 'CDK_OUTDIR';
 export const CONTEXT_ENV = 'CDK_CONTEXT_JSON';
 
@@ -38,15 +48,33 @@ export interface MissingContext {
   };
 }
 
-export interface SynthesizeResponse {
+export interface AssemblyManifest {
   /**
    * Protocol version
    */
   version: string;
-  stacks: SynthesizedStack[];
+
+  /**
+   * The set of artifacts in this assembly.
+   */
+  artifacts?: { [id: string]: Artifact };
+
+  /**
+   * Runtime information.
+   */
   runtime?: AppRuntime;
+
+  /**
+   * @deprecated stacks should be read from `Artifacts`.
+   */
+  stacks: SynthesizedStack[];
 }
 
+/**
+ * @deprecated use `AssemblyManifest`
+ */
+export type SynthesizeResponse = AssemblyManifest;
+
 /**
  * A complete synthesized stack
  */

packages/@aws-cdk/cx-api/lib/cxapi.ts

@@ -5,3 +5,4 @@ export * from './context/vpc';
 export * from './context/ssm-parameter';
 export * from './context/availability-zones';
 export * from './metadata/assets';
+export * from './artifacts';