env: fix a bunch of escaping/parsing edge cases (#308)

* Handle standard escape sequences based on XSI/XBD
  https://pubs.opengroup.org/onlinepubs/9699919799/utilities/echo.html
* Fix parsing with escaped quotes at end of quoted
  value
* Handle key whose name starts with `export` (require
  whitespace between `export` keyword when stripping)
* Fix parsing of unquoted values with embedded `#`
  (require a space before starting an inline comment)
* Use correct parser methods throughout all test cases
* Eliminate a lot of unused code that was not working
  right

Signed-off-by: Milas Bowman <[email protected]>

Author: milas

dotenv/godotenv.go

@@ -4,18 +4,17 @@
 //
 // The TL;DR is that you make a .env file that looks something like
 //
-// 		SOME_ENV_VAR=somevalue
+//	SOME_ENV_VAR=somevalue
 //
 // and then in your go code you can call
 //
-// 		godotenv.Load()
+//	godotenv.Load()
 //
 // and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR")
 package dotenv
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -28,10 +27,10 @@ import (
 	"github.com/compose-spec/compose-go/template"
 )
 
-const doubleQuoteSpecialChars = "\\\n\r\"!$`"
-
 var utf8BOM = []byte("\uFEFF")
 
+var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored
+
 // LookupFn represents a lookup function to resolve variables from
 type LookupFn func(string) (string, bool)
 
@@ -60,13 +59,13 @@ func ParseWithLookup(r io.Reader, lookupFn LookupFn) (map[string]string, error)
 
 // Load will read your env file(s) and load them into ENV for this process.
 //
-// Call this function as close as possible to the start of your program (ideally in main)
+// Call this function as close as possible to the start of your program (ideally in main).
 //
-// If you call Load without any args it will default to loading .env in the current path
+// If you call Load without any args it will default to loading .env in the current path.
 //
-// You can otherwise tell it which files to load (there can be more than one) like
+// You can otherwise tell it which files to load (there can be more than one) like:
 //
-//		godotenv.Load("fileone", "filetwo")
+//	godotenv.Load("fileone", "filetwo")
 //
 // It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
 func Load(filenames ...string) error {
@@ -75,13 +74,13 @@ func Load(filenames ...string) error {
 
 // Overload will read your env file(s) and load them into ENV for this process.
 //
-// Call this function as close as possible to the start of your program (ideally in main)
+// Call this function as close as possible to the start of your program (ideally in main).
 //
-// If you call Overload without any args it will default to loading .env in the current path
+// If you call Overload without any args it will default to loading .env in the current path.
 //
-// You can otherwise tell it which files to load (there can be more than one) like
+// You can otherwise tell it which files to load (there can be more than one) like:
 //
-//		godotenv.Overload("fileone", "filetwo")
+//	godotenv.Overload("fileone", "filetwo")
 //
 // It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars.
 func Overload(filenames ...string) error {
@@ -99,8 +98,6 @@ func load(overload bool, filenames ...string) error {
 	return nil
 }
 
-var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored
-
 // ReadWithLookup gets all env vars from the files and/or lookup function and return values as
 // a map rather than automatically writing values into env
 func ReadWithLookup(lookupFn LookupFn, filenames ...string) (map[string]string, error) {
@@ -155,6 +152,8 @@ func UnmarshalBytesWithLookup(src []byte, lookupFn LookupFn) (map[string]string,
 //
 // If you want more fine grained control over your command it's recommended
 // that you use `Load()` or `Read()` and the `os/exec` package yourself.
+//
+// Deprecated: Use the `os/exec` package directly.
 func Exec(filenames []string, cmd string, cmdArgs []string) error {
 	if err := Load(filenames...); err != nil {
 		return err
@@ -168,7 +167,10 @@ func Exec(filenames []string, cmd string, cmdArgs []string) error {
 }
 
 // Write serializes the given environment and writes it to a file
+//
+// Deprecated: The serialization functions are untested and unmaintained.
 func Write(envMap map[string]string, filename string) error {
+	//goland:noinspection GoDeprecation
 	content, err := Marshal(envMap)
 	if err != nil {
 		return err
@@ -187,6 +189,8 @@ func Write(envMap map[string]string, filename string) error {
 
 // Marshal outputs the given environment as a dotenv-formatted environment file.
 // Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped.
+//
+// Deprecated: The serialization functions are untested and unmaintained.
 func Marshal(envMap map[string]string) (string, error) {
 	lines := make([]string, 0, len(envMap))
 	for k, v := range envMap {
@@ -239,104 +243,6 @@ func readFile(filename string, lookupFn LookupFn) (map[string]string, error) {
 	return ParseWithLookup(file, lookupFn)
 }
 
-var exportRegex = regexp.MustCompile(`^\s*(?:export\s+)?(.*?)\s*$`)
-
-func parseLine(line string, envMap map[string]string) (string, string, error) {
-	return parseLineWithLookup(line, envMap, nil)
-}
-func parseLineWithLookup(line string, envMap map[string]string, lookupFn LookupFn) (string, string, error) {
-	if line == "" {
-		return "", "", errors.New("zero length string")
-	}
-
-	// ditch the comments (but keep quoted hashes)
-	if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.Contains(line, " #") {
-		segmentsBetweenHashes := strings.Split(line, "#")
-		quotesAreOpen := false
-		var segmentsToKeep []string
-		for _, segment := range segmentsBetweenHashes {
-			if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
-				if quotesAreOpen {
-					segmentsToKeep = append(segmentsToKeep, segment)
-				}
-				quotesAreOpen = !quotesAreOpen
-			}
-
-			if len(segmentsToKeep) == 0 || quotesAreOpen {
-				segmentsToKeep = append(segmentsToKeep, segment)
-			}
-		}
-
-		line = strings.Join(segmentsToKeep, "#")
-	}
-
-	firstEquals := strings.Index(line, "=")
-	firstColon := strings.Index(line, ":")
-	splitString := strings.SplitN(line, "=", 2)
-	if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) {
-		// This is a yaml-style line
-		splitString = strings.SplitN(line, ":", 2)
-	}
-
-	if len(splitString) != 2 {
-		return "", "", errors.New("can't separate key from value")
-	}
-	key := exportRegex.ReplaceAllString(splitString[0], "$1")
-
-	// Parse the value
-	value := parseValue(splitString[1], envMap, lookupFn)
-
-	return key, value, nil
-}
-
-var (
-	singleQuotesRegex  = regexp.MustCompile(`\A'(.*)'\z`)
-	doubleQuotesRegex  = regexp.MustCompile(`\A"(.*)"\z`)
-	escapeRegex        = regexp.MustCompile(`\\.`)
-	unescapeCharsRegex = regexp.MustCompile(`\\([^$])`)
-)
-
-func parseValue(value string, envMap map[string]string, lookupFn LookupFn) string {
-
-	// trim
-	value = strings.Trim(value, " ")
-
-	// check if we've got quoted values or possible escapes
-	if len(value) > 1 {
-		singleQuotes := singleQuotesRegex.FindStringSubmatch(value)
-
-		doubleQuotes := doubleQuotesRegex.FindStringSubmatch(value)
-
-		if singleQuotes != nil || doubleQuotes != nil {
-			// pull the quotes off the edges
-			value = value[1 : len(value)-1]
-		}
-
-		if doubleQuotes != nil {
-			// expand newlines
-			value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string {
-				c := strings.TrimPrefix(match, `\`)
-				switch c {
-				case "n":
-					return "\n"
-				case "r":
-					return "\r"
-				default:
-					return match
-				}
-			})
-			// unescape characters
-			value = unescapeCharsRegex.ReplaceAllString(value, "$1")
-		}
-
-		if singleQuotes == nil {
-			value, _ = expandVariables(value, envMap, lookupFn)
-		}
-	}
-
-	return value
-}
-
 func expandVariables(value string, envMap map[string]string, lookupFn LookupFn) (string, error) {
 	retVal, err := template.Substitute(value, func(k string) (string, bool) {
 		if v, ok := envMap[k]; ok {
@@ -350,7 +256,9 @@ func expandVariables(value string, envMap map[string]string, lookupFn LookupFn)
 	return retVal, nil
 }
 
+// Deprecated: only used by unsupported/untested code for Marshal/Write.
 func doubleQuoteEscape(line string) string {
+	const doubleQuoteSpecialChars = "\\\n\r\"!$`"
 	for _, c := range doubleQuoteSpecialChars {
 		toReplace := "\\" + string(c)
 		if c == '\n' {