Merge pull request #338 from ndeloof/ResolveServicesEnvironment

closes docker/compose#8713

Author: ndeloof

cli/options.go

@@ -167,6 +167,14 @@ func WithLoadOptions(loadOptions ...func(*loader.Options)) ProjectOptionsFn {
 	}
 }
 
+// WithProfiles sets profiles to be activated
+func WithProfiles(profiles []string) ProjectOptionsFn {
+	return func(o *ProjectOptions) error {
+		o.loadOptions = append(o.loadOptions, loader.WithProfiles(profiles))
+		return nil
+	}
+}
+
 // WithOsEnv imports environment variables from OS
 func WithOsEnv(o *ProjectOptions) error {
 	for k, v := range utils.GetAsEqualsMap(os.Environ()) {

loader/loader.go

@@ -17,9 +17,7 @@
 package loader
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	paths "path"
 	"path/filepath"
@@ -30,7 +28,6 @@ import (
 	"time"
 
 	"github.com/compose-spec/compose-go/consts"
-	"github.com/compose-spec/compose-go/dotenv"
 	interp "github.com/compose-spec/compose-go/interpolation"
 	"github.com/compose-spec/compose-go/schema"
 	"github.com/compose-spec/compose-go/template"
@@ -67,6 +64,8 @@ type Options struct {
 	projectName string
 	// Indicates when the projectName was imperatively set or guessed from path
 	projectNameImperativelySet bool
+	// Profiles set profiles to enable
+	Profiles []string
 }
 
 func (o *Options) SetProjectName(name string, imperativelySet bool) {
@@ -125,6 +124,13 @@ func WithSkipValidation(opts *Options) {
 	opts.SkipValidation = true
 }
 
+// WithProfiles sets profiles to be activated
+func WithProfiles(profiles []string) func(*Options) {
+	return func(opts *Options) {
+		opts.Profiles = profiles
+	}
+}
+
 // ParseYAML reads the bytes from a file, parses the bytes into a mapping
 // structure, and returns it.
 func ParseYAML(source []byte) (map[string]interface{}, error) {
@@ -198,12 +204,6 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 		if err != nil {
 			return nil, err
 		}
-		if opts.discardEnvFiles {
-			for i := range cfg.Services {
-				cfg.Services[i].EnvFile = nil
-			}
-		}
-
 		configs = append(configs, cfg)
 	}
 
@@ -246,7 +246,13 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 		}
 	}
 
-	return project, nil
+	if len(opts.Profiles) > 0 {
+		project.ApplyProfiles(opts.Profiles)
+	}
+
+	err = project.ResolveServicesEnvironment(opts.discardEnvFiles)
+
+	return project, err
 }
 
 func projectName(details types.ConfigDetails, opts *Options) (string, error) {
@@ -601,10 +607,6 @@ func LoadService(name string, serviceDict map[string]interface{}, workingDir str
 	}
 	serviceConfig.Name = name
 
-	if err := resolveEnvironment(serviceConfig, workingDir, lookupEnv); err != nil {
-		return nil, err
-	}
-
 	for i, volume := range serviceConfig.Volumes {
 		if volume.Type != types.VolumeTypeBind {
 			continue
@@ -641,52 +643,6 @@ func convertVolumePath(volume types.ServiceVolumeConfig) types.ServiceVolumeConf
 	return volume
 }
 
-func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string, lookupEnv template.Mapping) error {
-	environment := types.MappingWithEquals{}
-	var resolve dotenv.LookupFn = func(s string) (string, bool) {
-		if v, ok := environment[s]; ok && v != nil {
-			return *v, true
-		}
-		return lookupEnv(s)
-	}
-
-	if len(serviceConfig.EnvFile) > 0 {
-		if serviceConfig.Environment == nil {
-			serviceConfig.Environment = types.MappingWithEquals{}
-		}
-		for _, envFile := range serviceConfig.EnvFile {
-			filePath := absPath(workingDir, envFile)
-			file, err := os.Open(filePath)
-			if err != nil {
-				return err
-			}
-
-			b, err := io.ReadAll(file)
-			if err != nil {
-				return err
-			}
-
-			// Do not defer to avoid it inside a loop
-			file.Close() //nolint:errcheck
-
-			fileVars, err := dotenv.ParseWithLookup(bytes.NewBuffer(b), resolve)
-			if err != nil {
-				return errors.Wrapf(err, "Failed to load %s", filePath)
-			}
-			env := types.MappingWithEquals{}
-			for k, v := range fileVars {
-				v := v
-				env[k] = &v
-			}
-			environment.OverrideBy(env.Resolve(lookupEnv).RemoveEmpty())
-		}
-	}
-
-	environment.OverrideBy(serviceConfig.Environment.Resolve(lookupEnv))
-	serviceConfig.Environment = environment
-	return nil
-}
-
 func resolveMaybeUnixPath(path string, workingDir string, lookupEnv template.Mapping) string {
 	filePath := expandUser(path, lookupEnv)
 	// Check if source is an absolute path (either Unix or Windows), to

loader/loader_test.go

@@ -814,7 +814,9 @@ func TestDiscardEnvFileOption(t *testing.T) {
 	configDetails := buildConfigDetails(dict, nil)
 
 	// Default behavior keeps the `env_file` entries
-	configWithEnvFiles, err := Load(configDetails)
+	configWithEnvFiles, err := Load(configDetails, func(options *Options) {
+		options.SkipNormalization = true
+	})
 	assert.NilError(t, err)
 	assert.DeepEqual(t, configWithEnvFiles.Services[0].EnvFile, types.StringList{"example1.env",
 		"example2.env"})
@@ -1936,17 +1938,22 @@ func TestLoadServiceWithEnvFile(t *testing.T) {
 	_, err = file.Write([]byte("HALLO=$TEST"))
 	assert.NilError(t, err)
 
-	m := map[string]interface{}{
-		"env_file": file.Name(),
+	p := &types.Project{
+		Environment: map[string]string{
+			"TEST": "YES",
+		},
+		Services: []types.ServiceConfig{
+			{
+				Name:    "Test",
+				EnvFile: []string{file.Name()},
+			},
+		},
 	}
-	s, err := LoadService("Test Name", m, ".", func(s string) (string, bool) {
-		if s == "TEST" {
-			return "YES", true
-		}
-		return "", false
-	}, true, false)
+	err = p.ResolveServicesEnvironment(false)
+	assert.NilError(t, err)
+	service, err := p.GetService("Test")
 	assert.NilError(t, err)
-	assert.Equal(t, "YES", *s.Environment["HALLO"])
+	assert.Equal(t, "YES", *service.Environment["HALLO"])
 }
 
 func TestLoadServiceWithVolumes(t *testing.T) {

loader/normalize.go

@@ -85,6 +85,9 @@ func normalize(project *types.Project, resolvePaths bool) error {
 			}
 			s.Build.Args = s.Build.Args.Resolve(fn)
 		}
+		for j, f := range s.EnvFile {
+			s.EnvFile[j] = absPath(project.WorkingDir, f)
+		}
 		s.Environment = s.Environment.Resolve(fn)
 
 		err := relocateLogDriver(&s)

types/project.go

@@ -17,28 +17,31 @@
 package types
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
 
+	"github.com/compose-spec/compose-go/dotenv"
 	"github.com/distribution/distribution/v3/reference"
 	godigest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
 
 // Project is the result of loading a set of compose files
 type Project struct {
-	Name         string            `yaml:"name,omitempty" json:"name,omitempty"`
-	WorkingDir   string            `yaml:"-" json:"-"`
-	Services     Services          `json:"services"`
-	Networks     Networks          `yaml:",omitempty" json:"networks,omitempty"`
-	Volumes      Volumes           `yaml:",omitempty" json:"volumes,omitempty"`
-	Secrets      Secrets           `yaml:",omitempty" json:"secrets,omitempty"`
-	Configs      Configs           `yaml:",omitempty" json:"configs,omitempty"`
-	Extensions   Extensions        `yaml:",inline" json:"-"` // https://github.com/golang/go/issues/6213
-	ComposeFiles []string          `yaml:"-" json:"-"`
-	Environment  map[string]string `yaml:"-" json:"-"`
+	Name         string     `yaml:"name,omitempty" json:"name,omitempty"`
+	WorkingDir   string     `yaml:"-" json:"-"`
+	Services     Services   `json:"services"`
+	Networks     Networks   `yaml:",omitempty" json:"networks,omitempty"`
+	Volumes      Volumes    `yaml:",omitempty" json:"volumes,omitempty"`
+	Secrets      Secrets    `yaml:",omitempty" json:"secrets,omitempty"`
+	Configs      Configs    `yaml:",omitempty" json:"configs,omitempty"`
+	Extensions   Extensions `yaml:",inline" json:"-"` // https://github.com/golang/go/issues/6213
+	ComposeFiles []string   `yaml:"-" json:"-"`
+	Environment  Mapping    `yaml:"-" json:"-"`
 
 	// DisabledServices track services which have been disable as profile is not active
 	DisabledServices Services `yaml:"-" json:"-"`
@@ -353,3 +356,41 @@ func (p *Project) ResolveImages(resolver func(named reference.Named) (godigest.D
 	}
 	return eg.Wait()
 }
+
+// ResolveServicesEnvironment parse env_files set for services to resolve the actual environment map for services
+func (p Project) ResolveServicesEnvironment(discardEnvFiles bool) error {
+	for i, service := range p.Services {
+		service.Environment = service.Environment.Resolve(p.Environment.Resolve)
+
+		environment := MappingWithEquals{}
+		// resolve variables based on other files we already parsed, + project's environment
+		var resolve dotenv.LookupFn = func(s string) (string, bool) {
+			v, ok := environment[s]
+			if ok && v != nil {
+				return *v, ok
+			}
+			return p.Environment.Resolve(s)
+		}
+
+		for _, envFile := range service.EnvFile {
+			b, err := os.ReadFile(envFile)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to load %s", envFile)
+			}
+
+			fileVars, err := dotenv.ParseWithLookup(bytes.NewBuffer(b), resolve)
+			if err != nil {
+				return err
+			}
+			environment.OverrideBy(Mapping(fileVars).ToMappingWithEquals())
+		}
+
+		service.Environment = environment.OverrideBy(service.Environment)
+
+		if discardEnvFiles {
+			service.EnvFile = nil
+		}
+		p.Services[i] = service
+	}
+	return nil
+}

types/types.go

@@ -482,6 +482,21 @@ func NewMapping(values []string) Mapping {
 	return mapping
 }
 
+// ToMappingWithEquals converts Mapping into a MappingWithEquals with pointer references
+func (m Mapping) ToMappingWithEquals() MappingWithEquals {
+	mapping := MappingWithEquals{}
+	for k, v := range m {
+		v := v
+		mapping[k] = &v
+	}
+	return mapping
+}
+
+func (m Mapping) Resolve(s string) (string, bool) {
+	v, ok := m[s]
+	return v, ok
+}
+
 // Labels is a mapping type for labels
 type Labels map[string]string