ARC-736 improving .env file structure (#1372)

* ARC-1508 - GHE UI: Update manual form page (#1363)

* - Updated the order of inputs in the manual form
- Text changes

* - Reverted

* using .env for docker-compose variables with authtoken

* only create .env file in non-prod envs

* updating postmerge to move global values back to .env

* adding example dev local

* latest work on creating default .env file, updating contributing docs

* adding better documentation

Co-authored-by: kAy <[email protected]>

Author: mboudreau

Diff for: .dockerignore

@@ -1,9 +1,13 @@
 # Environment Files
 .env
-.envrc
+.env.local
+.env.local.*
+.env.*.local
+.env.*.local.*
 
 # DB Secret File
 *.pem
+**/*.pem
 
 # Transpiled files
 build/

Diff for: .env.example renamed to .env.development

@@ -1,11 +1,6 @@
 # APP VARIABLES - NEEDED FOR PRODUCTION
 # The ID of your GitHub App
 NODE_ENV=development
-APP_ID=
-APP_URL=
-WEBHOOK_SECRET=development
-GITHUB_CLIENT_ID=
-GITHUB_CLIENT_SECRET=
 SQS_BACKFILL_QUEUE_URL=http://127.0.0.1:4566/000000000000/backfill
 SQS_BACKFILL_QUEUE_REGION=us-west-1
 SQS_PUSH_QUEUE_URL=http://127.0.0.1:4566/000000000000/push
@@ -15,32 +10,18 @@ SQS_DEPLOYMENT_QUEUE_REGION=us-west-1
 SQS_BRANCH_QUEUE_URL=http://127.0.0.1:4566/000000000000/branch
 SQS_BRANCH_QUEUE_REGION=us-west-1
 MICROS_AWS_REGION=us-west-1
-# Optional but needed for feature flags
-LAUNCHDARKLY_KEY=
-
-# Unique name for the jira app, used in the Atlassian Connect manifest to
-# differentiate this instance from other deployments (staging, dev instances, etc).
-INSTANCE_NAME=<jira-app-name>
+GLOBAL_HASH_SECRET=testsecret
 
 # The Postgres URL used to connect to the database and secret for encrypting data
 DATABASE_URL=postgres://postgres:[email protected]:5432/jira-dev
-# Generate using `openssl rand -hex 32`
-STORAGE_SECRET=
+STORAGE_SECRET=1a5ce262945c691b1da27ed704a6c694f6eb6d8e15bc1612e2460d1aa76ce8fd
 
 # DEVELOPMENT VARIABLES
 ATLASSIAN_SECRET=development-secret
 AWS_ACCESS_KEY_ID=development
 AWS_SECRET_ACCESS_KEY=development
-TUNNEL_PORT=8080
-WORKER_PORT=8081
-TUNNEL_SUBDOMAIN=<subdomain>
-PRIVATE_KEY_PATH=<your-private-key>.pem
-ATLASSIAN_URL=https://<your-instance>.atlassian.net
-NGROK_AUTHTOKEN=<your-ngrok-authtoken>
-
-# Use `trace` to get verbose logging or `info` to show less
+MAINTENANCE_MODE=false
 LOG_LEVEL=debug
-WEBHOOK_PROXY_URL=
 
 # Cryptor
 CRYPTOR_URL=http://cryptor-sidecar:26272

Diff for: .env.development.local-example

@@ -0,0 +1,16 @@
+##### DUPLICATE THIS FILE AND REMOVE THE `-example` AT THE END #####
+
+# APP VARIABLES - NEEDED FOR PRODUCTION
+# Github App Information
+APP_ID=
+WEBHOOK_SECRET=
+GITHUB_CLIENT_ID=
+GITHUB_CLIENT_SECRET=
+# Path to github private key file (relative to root of this project). If at the root, just specify filename
+PRIVATE_KEY_PATH=
+
+# Unique name for the jira app, used in the Atlassian Connect manifest to
+# differentiate this instance from other deployments (staging, dev instances, etc).
+INSTANCE_NAME=
+# The Jira Instance URL you created for this app (ex. https://example.atlassian.net)
+ATLASSIAN_URL=

Diff for: .gitignore

@@ -2,12 +2,15 @@
 db/config.json
 
 # Environment Files
-.env*
-!.env.example
-!.env.test
+.env
+.env.local
+.env.local.*
+.env.*.local
+.env.*.local.*
 
 # DB Secret File
 *.pem
+**/*.pem
 
 # Transpiled files
 build/

Diff for: .husky/create-env.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+DIR=$(dirname "$0")
+FILE="${DIR}/../.env"
+
+if [ !  -f "$FILE" ]; then
+  echo ".env file not found, creating it..."
+  touch "$FILE"
+  echo "APP_URL=http://localhost" >> "$FILE"
+  echo "WEBHOOK_PROXY_URL=http://localhost/github/events" >> "$FILE"
+  echo "NGROK_AUTHTOKEN=insert ngrok token here" >> "$FILE"
+  echo ".env file created with defaults"
+fi

Diff for: .husky/post-merge

@@ -0,0 +1,17 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+DIR=$(dirname "$0")
+OLDFILE="${DIR}/../.env"
+NEWFILE="${DIR}/../.env.development.local"
+REGEX="^(APP_URL|WEBHOOK_PROXY_URL|NGROK_AUTHTOKEN)=.*"
+if [ -f "$OLDFILE" ] && [ !  -f "$NEWFILE" ]; then
+  echo "Detected old .env file, moving to .env.development.local"
+  mv "$OLDFILE" "$NEWFILE"
+  echo "Moving global values back to .env file"
+  cat "$NEWFILE" | grep -oP "${REGEX}" > "$OLDFILE"
+  echo "Removing moved from .env.development.local file"
+  sed -i -E "s/${REGEX}//" "$NEWFILE"
+fi
+
+. "$(dirname "$0")/create-env.sh"

Diff for: CONTRIBUTING.md

@@ -65,19 +65,20 @@ Your new GitHub app will need the following repository permissions & events:
 + Repository
 + Workflow run
 
-### Setting up your `.env` file
+### Setting up your environment file
 
-Once you've set up your GitHub app and cloned this repo, copy the content from `.env.example` and paste it to a new file called `.env`, with the following configuration:
+The environment files work in a fairly standardized way of having a "global" `.env` that holds information needed across all environments but is not committed. Then we have `NODE_ENV` specific environment files like `.env.development`, `.env.test`, etc, as they are non-sensitive default variables needed for those environments to work.  Since they are committed, please be careful not to add sensitive information to these files - if you need to add sensitive information or you want to overwrite the environment defaults, you can create a `.local` version of that file and that will never be committed. 
+
+Once you've set up your GitHub app and cloned this repo, copy the file `.env.development.local-example` to a new file called `.env.development.local`.  Fill in the blank fields in the file:
 
 + `APP_ID` and `GITHUB_CLIENT_ID`: Copy these values over from your new GitHub app page.
-+ `APP_URL`: `https://DOMAIN`
 + `GITHUB_CLIENT_SECRET`: You'll need to generate a new one on your GitHub app page by hitting the `Generate a new client secret` button. Copy and paste the generated secret.
-+ `TUNNEL_SUBDOMAIN`: the subdomain you want to use to allow access from the internet to your local machine
 + `PRIVATE_KEY_PATH`: You'll also need to generate a new private key on your GitHub app page, download it, move it to the source root of this repo, and set `PRIVATE_KEY_PATH=<your-private-key-name>.pem`
-+ `ATLASSIAN_URL`: The URL for the Jira instance you're testing it. If you don't have one now, please set the value of this variable after going through the step 1 of "Configuring the Jira instance" section of this document.
-+ `STORAGE_SECRET`: It needs to be set to a 32 char secret (anything else fails). You can generate one by running `openssl rand -hex 32` in your terminal and paste directly to your .env file.
-+ `INSTANCE_NAME`: Your Jira app name - will show as "GitHub (instance-name)"
-+ `WEBHOOK_PROXY_URL`: `https://DOMAIN/github/events`
++ `ATLASSIAN_URL`: The URL for the Jira instance you're testing on. If you don't have one now, [please set the value of this variable from the steps mentioned here](#create-your-jira-instance).
++ `INSTANCE_NAME`: Your Jira app name - will show as "GitHub for Jira (instance-name)"
+
+Lastly, you need to replace the value of the follow variables in the global `.env` file:
+
 + `NGROK_AUTHTOKEN`: Your ngrok authtoken.  If you want to use ngrok as a tunnel to test it on your Jira instance, you need an authtoken. Simply [login/signup to ngrok](https://dashboard.ngrok.com/get-started/setup), copy & paste the authtoken into this var.
 
 ### Running the app
@@ -87,7 +88,6 @@ The first time you run the app, simply run:
 ```
 yarn install # installs node modules
 docker-compose up # Spin up docker containers
-yarn run init # Creates DBs and initializes tables, creates sqs queues
 ```
 
 That's it.  Everything is ran in docker-compose, including redis, postgres, ngrok and the app (main and worker thread).

Diff for: package.json

@@ -12,31 +12,30 @@
 		"prestart": "run-p setup clean db:dev",
 		"init": "run-p db",
 		"start": "run-p start:main start:worker",
-		"start:main": "tsnd -r tsconfig-paths/register --watch=.env,db/config.json,**/*.hbs --inspect=0.0.0.0:9229 --respawn --transpile-only -- src/main.ts",
-		"start:worker": "tsnd -r tsconfig-paths/register --watch=.env,db/config.json,**/*.hbs --inspect=0.0.0.0:9230 --respawn --transpile-only -- src/worker.ts",
+		"start:main": "tsnd -r tsconfig-paths/register --watch=.env*,db/config.json,**/*.hbs --inspect=0.0.0.0:9229 --respawn --transpile-only -- src/main.ts",
+		"start:worker": "tsnd -r tsconfig-paths/register --watch=.env*,db/config.json,**/*.hbs --inspect=0.0.0.0:9230 --respawn --transpile-only -- src/worker.ts",
 		"start:production": "run-p start:main:production start:worker:production",
 		"start:main:production": "run-s db:migrate:prod && node -r tsconfig-paths/register src/main",
 		"start:worker:production": "run-s db:migrate:prod && node -r tsconfig-paths/register src/worker",
 		"clean": "rimraf coverage build tmp src/**/*.js src/**/*.js.map test/**/*.js test/**/*.js.map",
-		"setup": "ts-node -r tsconfig-paths/register prestart.ts",
-		"setup:test": "NODE_ENV=test ts-node -r tsconfig-paths/register prestart.ts",
+		"setup": "dotenv -- ts-node -r tsconfig-paths/register prestart.ts",
 		"build": "tsc -p ./tsconfig.json",
 		"build:release": "tsc -p ./tsconfig.release.json",
 		"build:watch": "tsc -w -p ./tsconfig.json",
 		"lint": "run-p lint:*",
 		"lint:ts": "eslint . --ext .ts,.tsx",
 		"lint:yaml": "yamllint github-for-jira.sd.yml",
-		"pretest": "run-p clean setup:test db:test",
+		"pretest": "NODE_ENV=test run-p clean setup db:test",
 		"test": "jest --runInBand --forceExit --coverage",
 		"test:watch": "jest --runInBand --watchAll",
 		"db": "run-s db:create db:migrate",
 		"db:create": "run-p db:create:dev db:create:test",
-		"db:create:dev": "dotenv -- sequelize db:create || exit 0",
-		"db:create:test": "dotenv -e .env.test -- sequelize db:create --env test || exit 0",
+		"db:create:dev": "dotenv -c development -- sequelize db:create || exit 0",
+		"db:create:test": "dotenv -c test -- sequelize db:create --env test || exit 0",
 		"db:create:prod": "sequelize db:create --env production-migrate || exit 0",
 		"db:migrate": "run-p db:migrate:dev db:migrate:test",
-		"db:migrate:dev": "dotenv -- sequelize db:migrate",
-		"db:migrate:test": "dotenv -e .env.test -- sequelize db:migrate --env test",
+		"db:migrate:dev": "dotenv -c development -- sequelize db:migrate",
+		"db:migrate:test": "dotenv -c test -- sequelize db:migrate --env test",
 		"db:migrate:prod": "sequelize db:migrate --env production-migrate",
 		"db:dev": "run-s db:create:dev db:migrate:dev",
 		"db:test": "run-s db:create:test db:migrate:test",
@@ -61,6 +60,8 @@
 		"cookie-session": "^2.0.0",
 		"csurf": "^1.11.0",
 		"date-fns": "^1.29.0",
+		"dotenv": "^16.0.1",
+		"dotenv-expand": "^8.0.3",
 		"express": "^4.17.3",
 		"express-rate-limit": "^5.1.3",
 		"express-sslify": "^1.2.0",
@@ -109,7 +110,7 @@
 		"@types/uuid": "^8.3.4",
 		"@typescript-eslint/eslint-plugin": "~5.12.0",
 		"@typescript-eslint/parser": "~5.12.0",
-		"dotenv-cli": "^4.0.0",
+		"dotenv-cli": "^6.0.0",
 		"eslint": "~8.9.0",
 		"eslint-config-prettier": "~8.3.0",
 		"eslint-import-resolver-typescript": "^2.5.0",

Diff for: prestart.ts

@@ -1,10 +1,11 @@
-import { envVars }  from "./src/config/env";
 import axios, { AxiosResponse } from "axios";
 import * as fs from "fs";
 import * as path from "path";
-import { isNodeDev } from "./src/util/is-node-env";
+import { exec } from "child_process";
+import { isNodeProd } from "./src/util/is-node-env";
 
-const envFilePath = path.resolve(__dirname, ".env");
+const envFileName = ".env";
+const envFilePath = path.resolve(__dirname, envFileName);
 
 const callTunnel = async () => {
 	const results = await Promise.all([
@@ -22,12 +23,6 @@ const wait = async (time = 0) => {
 };
 
 const waitForTunnel = async () => {
-	// Does .env exist?
-	if (isNodeDev() && !fs.existsSync(envFilePath)) {
-		console.error(`.env file doesn't exist. Please create it following the steps in the CONTRIBUTING.md file.`);
-		process.exit(1);
-	}
-
 	// Call the service 3 times until ready
 	const response = await callTunnel()
 		.catch(callTunnel)
@@ -42,22 +37,22 @@ const waitForTunnel = async () => {
 			envContents = envContents.replace(/APP_URL=.*/, `APP_URL=${ngrokDomain}`);
 			envContents = envContents.replace(/WEBHOOK_PROXY_URL=.*/, `WEBHOOK_PROXY_URL=${ngrokDomain}/github/events`);
 			fs.writeFileSync(envFilePath, envContents);
-			console.info(`Updated .env file to use ngrok domain ${ngrokDomain}.`);
+			console.info(`Updated ${envFileName} file to use ngrok domain '${ngrokDomain}'.`);
 		} catch (e) {
 			console.info(`'${envFilePath}' not found, skipping...`);
 		}
 	} else {
-		console.info("Ngrok not running, skipping updating .env file.");
+		console.info(`Ngrok not running, skipping updating ${envFileName} file.`);
 	}
 };
 
 const callQueues = async () => {
-	const URLs = Object.keys(envVars)
-		.filter(key => /^SQS_.*_QUEUE_URL$/.test(key))
-		.map(key => envVars[key]);
+	const queueUrls = Object.entries(process.env)
+		.filter(([key, value]) => /^SQS_.*_QUEUE_URL$/.test(key) && value)
+		.map(([_key, value]) => value as string);
 	console.info(`Checking for localstack initialization...`);
-	if (URLs.length) {
-		const url = new URL(URLs[0]);
+	if (queueUrls.length) {
+		const url = new URL(queueUrls[0]);
 		const response = await axios.get(`${url.protocol}//${url.host}/health`, { responseType: "json" });
 		if (response.data?.services?.sqs !== "running") {
 			console.info("localstack not initialized.");
@@ -66,9 +61,9 @@ const callQueues = async () => {
 		console.info(`localstack initialized.`);
 	}
 
-	console.info(`Calling queues: ${URLs.join(", ")}`);
+	console.info(`Calling queues: ${queueUrls.join(", ")}`);
 	return Promise.all(
-		URLs.map(async (value) => {
+		queueUrls.map(async (value) => {
 			try {
 				await axios.get(value);
 				console.info(`Queue ${value} is ready...`);
@@ -92,10 +87,25 @@ const waitForQueues = async () => {
 	console.info("All queues ready.");
 };
 
+const createEnvFile = async () => {
+	if (!isNodeProd() && !fs.existsSync(envFilePath)) {
+		return new Promise<void>((resolve, reject) => {
+			exec("./.husky/create-env.sh", (error, stdout) => {
+				if (error) {
+					reject(error);
+					return;
+				}
+				console.info(stdout);
+				resolve();
+			});
+		})
+	}
+};
+
 // Check to see if ngrok is up and running
 (async function main() {
-
 	try {
+		await createEnvFile();
 		await Promise.all([
 			waitForTunnel(),
 			waitForQueues()

Diff for: src/config/env.ts

@@ -1,7 +1,8 @@
-import dotenv from "dotenv";
+import { config } from "dotenv";
+import { expand } from "dotenv-expand";
 import path from "path";
 import { LogLevelString } from "bunyan";
-import { getNodeEnv, isNodeTest } from "utils/is-node-env";
+import { getNodeEnv } from "utils/is-node-env";
 import { EnvironmentEnum } from "interfaces/common";
 
 const nodeEnv: EnvironmentEnum = EnvironmentEnum[getNodeEnv()];
@@ -26,15 +27,16 @@ const requiredEnvVars = [
 	"CRYPTOR_SIDECAR_CLIENT_IDENTIFICATION_CHALLENGE"
 ];
 
-const filename = isNodeTest() ? ".env.test" : ".env";
-const env = dotenv.config({
-	path: path.resolve(process.cwd(), filename)
-});
-
-// TODO: add checks for environment variables here and error out if missing any
-if (env.error && nodeEnv !== EnvironmentEnum.production) {
-	throw env.error;
-}
+// Load environment files
+const envFile = ".env";
+[
+	`${envFile}.${nodeEnv}.local`,
+	`${envFile}.local`,
+	`${envFile}.${nodeEnv}`,
+	envFile
+].map((env) => expand(config({
+	path: path.resolve(__dirname, "../..", env)
+})));
 
 const getProxyFromEnvironment = (): string | undefined => {
 	const proxyHost = process.env.EXTERNAL_ONLY_PROXY_HOST;
@@ -65,7 +67,7 @@ export interface EnvVars {
 	NODE_ENV: EnvironmentEnum,
 	MICROS_ENV: EnvironmentEnum;
 	MICROS_SERVICE_VERSION?: string;
-  MICROS_GROUP: string;
+	MICROS_GROUP: string;
 	SQS_BACKFILL_QUEUE_URL: string;
 	SQS_BACKFILL_QUEUE_REGION: string;
 	SQS_PUSH_QUEUE_URL: string;