feat: Model importer script (#118)

* Update package generator to be more explicit about importing (and to allow overwriting)

* Regenerate SDK packages

* Add command for importing multiple models by glob patterns

* Add script to copy model artifacts into repository

* Update ImportModels command description

Author: jeskew

.gitignore

@@ -21,3 +21,5 @@ jspm_packages
 
 lerna-debug.log
 package-lock.json
+
+!serviceModels/logs

package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "bootstrap": "lerna bootstrap",
     "clean": "npm run clear-build-cache && lerna clean",
-    "clear-build-cache": "lerna exec -- rm -rf ./build",
+    "clear-build-cache": "rimraf ./packages/*/build/*",
+    "copy-models": "node ./scripts/copyModels.js",
     "pretest": "lerna run pretest",
     "test": "jest --coverage"
   },
@@ -18,9 +19,12 @@
   "author": "[email protected]",
   "license": "UNLICENSED",
   "dependencies": {
+    "glob": "^7.1.2",
     "jest": "^20.0.4",
     "lerna": "^2.0.0",
-    "typescript": "^2.6"
+    "rimraf": "^2.6.2",
+    "typescript": "^2.6",
+    "yargs": "^8.0.2"
   },
   "workspaces": [
     "packages/*"

packages/package-generator/package.json

@@ -27,13 +27,17 @@
         "@aws/service-model": "^0.0.1",
         "@aws/service-types-generator": "^0.0.1",
         "@aws/types": "^0.0.1",
+        "glob": "^7.1.2",
+        "rimraf": "^2.6.2",
         "semver": "^5.4.1",
-        "yargs": "^8.0.2",
-        "tslib": "^1.8.0"
+        "tslib": "^1.8.0",
+        "yargs": "^8.0.2"
     },
     "devDependencies": {
+        "@types/glob": "^5.0.35",
         "@types/jest": "^20.0.2",
         "@types/node": "^7.0.12",
+        "@types/rimraf": "^2.0.2",
         "@types/semver": "^5.4.0",
         "@types/yargs": "^8.0",
         "jest": "^20.0.4",

packages/package-generator/src/ClientModuleGenerator.ts

@@ -1,3 +1,4 @@
+import { clientModuleIdentifier } from "./clientModuleIdentifier";
 import { ModuleGenerator } from "./ModuleGenerator";
 import {
   ConfigurationDefinition,
@@ -21,6 +22,7 @@ export interface ClientModuleInit {
   customizations?: Array<CustomizationDefinition>;
   model: TreeModel;
   runtime: RuntimeTarget;
+  prefix?: string;
   version?: string;
 }
 
@@ -33,20 +35,12 @@ export class ClientModuleGenerator extends ModuleGenerator {
   constructor({
     customizations,
     model,
+    prefix = "",
     runtime,
     version = "0.0.1"
   }: ClientModuleInit) {
-    let name = `sdk-${getServiceId(model.metadata)}`;
-    const modelVersion = determineServiceVersion(model.metadata);
-    if (modelVersion > 1) {
-      name += `-v${modelVersion}`;
-    }
-    if (runtime !== "universal") {
-      name += `-${runtime}`;
-    }
-
     super({
-      name,
+      name: `${prefix}${clientModuleIdentifier(model.metadata, runtime)}`,
       description: `${runtime.substring(0, 1).toUpperCase()}${runtime.substring(
         1
       )} SDK for ${model.metadata.serviceFullName}`,
@@ -119,12 +113,15 @@ tsconfig.test.json
   }
 
   protected tsconfig() {
-    const { compilerOptions, ...rest } = super.tsconfig();
+    let { compilerOptions, ...rest } = super.tsconfig();
     if (
       compilerOptions.lib &&
       (this.target === "browser" || this.target === "universal")
     ) {
-      compilerOptions.lib.push("dom");
+      compilerOptions = {
+        ...compilerOptions,
+        lib: [...compilerOptions.lib, "dom"]
+      };
     }
 
     return {
@@ -211,39 +208,3 @@ tsconfig.test.json
 }
 
 type ObjStrMap = { [key: string]: string };
-
-function getServiceId(metadata: ServiceMetadata): string {
-  const { serviceAbbreviation, serviceFullName, serviceId } = metadata;
-
-  const className =
-    serviceId ||
-    (serviceAbbreviation || serviceFullName)
-      .replace(/^(aws|amazon)/i, "")
-      .trim();
-
-  return className.toLowerCase().replace(/\s/g, "-");
-}
-
-// TODO use metadata.major_version when added to the model
-function determineServiceVersion(metadata: ServiceMetadata): number {
-  const serviceId = getServiceId(metadata);
-  if (
-    serviceMajorVersions[serviceId] &&
-    serviceMajorVersions[serviceId][metadata.apiVersion]
-  ) {
-    return serviceMajorVersions[serviceId][metadata.apiVersion];
-  }
-
-  return 1;
-}
-
-interface MajorVersionMatcher {
-  [serviceIdentifier: string]: {
-    [apiVersion: string]: number;
-  };
-}
-const serviceMajorVersions: MajorVersionMatcher = {
-  dynamodb: {
-    "2012-08-10": 2
-  }
-};

packages/package-generator/src/ModuleGenerator.spec.ts

@@ -31,7 +31,7 @@ describe("ModuleGenerator", () => {
   });
 
   describe("package.json", () => {
-    it("should use the provided package name under the @aws scope", () => {
+    it("should use the provided package name", () => {
       const name = "name";
       const generator = new ModuleGenerator({
         name,
@@ -43,7 +43,7 @@ describe("ModuleGenerator", () => {
           found = true;
           const packageJson = JSON.parse(contents);
           expect(typeof packageJson).toBe("object");
-          expect(packageJson.name).toBe(`@aws/${name}`);
+          expect(packageJson.name).toBe(name);
         }
       }
 

packages/package-generator/src/ModuleGenerator.ts

@@ -44,7 +44,7 @@ export class ModuleGenerator {
 
   protected packageJson(): JsonDocument {
     return {
-      name: `@aws/${this.name}`,
+      name: this.name,
       description: this.description,
       version: this.version,
       scripts: {

packages/package-generator/src/cli.ts

@@ -1,10 +1,12 @@
 import {
-  CreateClientPackageCommand,
-  CreateCustomPackageCommand
+  CreateCustomPackageCommand,
+  ImportClientPackageCommand,
+  ImportModelsCommand
 } from "./commands";
 
 require("yargs")
   .command(CreateCustomPackageCommand)
-  .command(CreateClientPackageCommand)
+  .command(ImportClientPackageCommand)
+  .command(ImportModelsCommand)
   .demandCommand()
   .help().argv;

packages/package-generator/src/clientModuleIdentifier.spec.ts

@@ -0,0 +1,59 @@
+import { clientModuleIdentifier } from "./clientModuleIdentifier";
+import { ServiceMetadata } from "@aws/types";
+
+const minimalMetadata: ServiceMetadata = {
+  apiVersion: "2018-02-06",
+  endpointPrefix: "foo",
+  protocol: "json",
+  serviceFullName: "Amazon Simple Foo Service",
+  signatureVersion: "none",
+  uid: "foo-2018-02-06"
+};
+
+describe("clientModuleIdentifier", () => {
+  it("should use the service ID when available", () => {
+    expect(
+      clientModuleIdentifier({
+        ...minimalMetadata,
+        serviceId: "Foo Control Plane"
+      })
+    ).toBe("sdk-foo-control-plane");
+  });
+
+  it("should use the service abbreviation in the absence of a service ID", () => {
+    expect(
+      clientModuleIdentifier({
+        ...minimalMetadata,
+        serviceAbbreviation: "SFS"
+      })
+    ).toBe("sdk-sfs");
+  });
+
+  it("should fall back to the service full name", () => {
+    expect(clientModuleIdentifier(minimalMetadata)).toBe(
+      "sdk-simple-foo-service"
+    );
+  });
+
+  it("should apply known version identifiers", () => {
+    expect(
+      clientModuleIdentifier({
+        ...minimalMetadata,
+        serviceId: "DynamoDB",
+        apiVersion: "2012-08-10"
+      })
+    ).toBe("sdk-dynamodb-v2");
+  });
+
+  it("should append the runtime target", () => {
+    expect(clientModuleIdentifier(minimalMetadata, "node")).toBe(
+      "sdk-simple-foo-service-node"
+    );
+  });
+
+  it("should omit the runtime for universal packages", () => {
+    expect(clientModuleIdentifier(minimalMetadata, "universal")).toBe(
+      "sdk-simple-foo-service"
+    );
+  });
+});

packages/package-generator/src/clientModuleIdentifier.ts

@@ -0,0 +1,55 @@
+import { RuntimeTarget } from "@aws/build-types";
+import { ServiceMetadata } from "@aws/types";
+
+export function clientModuleIdentifier(
+  metadata: ServiceMetadata,
+  runtime: RuntimeTarget = "universal"
+): string {
+  let name = `sdk-${getServiceId(metadata)}`;
+  const modelVersion = determineServiceVersion(metadata);
+  if (modelVersion > 1) {
+    name += `-v${modelVersion}`;
+  }
+  if (runtime !== "universal") {
+    name += `-${runtime}`;
+  }
+
+  return name;
+}
+
+function getServiceId(metadata: ServiceMetadata): string {
+  const { serviceAbbreviation, serviceFullName, serviceId } = metadata;
+
+  const className =
+    serviceId ||
+    (serviceAbbreviation || serviceFullName)
+      .replace(/^(aws|amazon)/i, "")
+      .trim();
+
+  return className.toLowerCase().replace(/\s/g, "-");
+}
+
+// TODO use metadata.major_version when added to the model
+function determineServiceVersion(metadata: ServiceMetadata): number {
+  const serviceId = getServiceId(metadata);
+  if (
+    serviceMajorVersions[serviceId] &&
+    serviceMajorVersions[serviceId][metadata.apiVersion]
+  ) {
+    return serviceMajorVersions[serviceId][metadata.apiVersion];
+  }
+
+  return 1;
+}
+
+interface MajorVersionMatcher {
+  [serviceIdentifier: string]: {
+    [apiVersion: string]: number;
+  };
+}
+
+const serviceMajorVersions: MajorVersionMatcher = {
+  dynamodb: {
+    "2012-08-10": 2
+  }
+};

packages/package-generator/src/commands/CreateClientPackageCommand.ts

@@ -17,7 +17,8 @@ export const CreateCustomPackageCommand: yargs.CommandModule = {
     name: {
       alias: ["n"],
       type: "string",
-      demandOption: true
+      demandOption: true,
+      coerce: name => `@aws/${name.replace(/^@aws\//, "")}`
     },
     description: {
       alias: ["d"],