Merge pull request #344 from glours/replace-compose-ref-by-compose

Signed-off-by: Nicolas De Loof <[email protected]>

Author: ndeloof

README.md

@@ -7,7 +7,7 @@ Go reference library for parsing and loading Compose files as specified by the
 
 ## Used by
 
-* [compose-ref](https://github.com/compose-spec/compose-ref)
+* [compose](https://github.com/docker/compose)
 * [containerd/nerdctl](https://github.com/containerd/nerdctl)
 * [compose-cli](https://github.com/docker/compose-cli)
 * [tilt.dev](https://github.com/tilt-dev/tilt)

cli/options.go

@@ -167,6 +167,14 @@ func WithLoadOptions(loadOptions ...func(*loader.Options)) ProjectOptionsFn {
 	}
 }
 
+// WithProfiles sets profiles to be activated
+func WithProfiles(profiles []string) ProjectOptionsFn {
+	return func(o *ProjectOptions) error {
+		o.loadOptions = append(o.loadOptions, loader.WithProfiles(profiles))
+		return nil
+	}
+}
+
 // WithOsEnv imports environment variables from OS
 func WithOsEnv(o *ProjectOptions) error {
 	for k, v := range utils.GetAsEqualsMap(os.Environ()) {

loader/loader.go

@@ -17,9 +17,7 @@
 package loader
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	paths "path"
 	"path/filepath"
@@ -30,7 +28,6 @@ import (
 	"time"
 
 	"github.com/compose-spec/compose-go/consts"
-	"github.com/compose-spec/compose-go/dotenv"
 	interp "github.com/compose-spec/compose-go/interpolation"
 	"github.com/compose-spec/compose-go/schema"
 	"github.com/compose-spec/compose-go/template"
@@ -59,6 +56,8 @@ type Options struct {
 	SkipConsistencyCheck bool
 	// Skip extends
 	SkipExtends bool
+	// Skip loading of env_files
+	SkipResolveEnvFiles bool
 	// Interpolation options
 	Interpolate *interp.Options
 	// Discard 'env_file' entries after resolving to 'environment' section
@@ -67,6 +66,8 @@ type Options struct {
 	projectName string
 	// Indicates when the projectName was imperatively set or guessed from path
 	projectNameImperativelySet bool
+	// Profiles set profiles to enable
+	Profiles []string
 }
 
 func (o *Options) SetProjectName(name string, imperativelySet bool) {
@@ -125,6 +126,13 @@ func WithSkipValidation(opts *Options) {
 	opts.SkipValidation = true
 }
 
+// WithProfiles sets profiles to be activated
+func WithProfiles(profiles []string) func(*Options) {
+	return func(opts *Options) {
+		opts.Profiles = profiles
+	}
+}
+
 // ParseYAML reads the bytes from a file, parses the bytes into a mapping
 // structure, and returns it.
 func ParseYAML(source []byte) (map[string]interface{}, error) {
@@ -195,12 +203,6 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 		if err != nil {
 			return nil, err
 		}
-		if opts.discardEnvFiles {
-			for i := range cfg.Services {
-				cfg.Services[i].EnvFile = nil
-			}
-		}
-
 		configs = append(configs, cfg)
 	}
 
@@ -243,7 +245,15 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 		}
 	}
 
-	return project, nil
+	if len(opts.Profiles) > 0 {
+		project.ApplyProfiles(opts.Profiles)
+	}
+
+	if !opts.SkipResolveEnvFiles {
+		err = project.ResolveServicesEnvironment(opts.discardEnvFiles)
+	}
+
+	return project, err
 }
 
 func projectName(details types.ConfigDetails, opts *Options) string {
@@ -587,10 +597,6 @@ func LoadService(name string, serviceDict map[string]interface{}, workingDir str
 	}
 	serviceConfig.Name = name
 
-	if err := resolveEnvironment(serviceConfig, workingDir, lookupEnv); err != nil {
-		return nil, err
-	}
-
 	for i, volume := range serviceConfig.Volumes {
 		if volume.Type != types.VolumeTypeBind {
 			continue

loader/loader_test.go

@@ -814,7 +814,9 @@ func TestDiscardEnvFileOption(t *testing.T) {
 	configDetails := buildConfigDetails(dict, nil)
 
 	// Default behavior keeps the `env_file` entries
-	configWithEnvFiles, err := Load(configDetails)
+	configWithEnvFiles, err := Load(configDetails, func(options *Options) {
+		options.SkipNormalization = true
+	})
 	assert.NilError(t, err)
 	assert.DeepEqual(t, configWithEnvFiles.Services[0].EnvFile, types.StringList{"example1.env",
 		"example2.env"})
@@ -1936,17 +1938,22 @@ func TestLoadServiceWithEnvFile(t *testing.T) {
 	_, err = file.Write([]byte("HALLO=$TEST"))
 	assert.NilError(t, err)
 
-	m := map[string]interface{}{
-		"env_file": file.Name(),
+	p := &types.Project{
+		Environment: map[string]string{
+			"TEST": "YES",
+		},
+		Services: []types.ServiceConfig{
+			{
+				Name:    "Test",
+				EnvFile: []string{file.Name()},
+			},
+		},
 	}
-	s, err := LoadService("Test Name", m, ".", func(s string) (string, bool) {
-		if s == "TEST" {
-			return "YES", true
-		}
-		return "", false
-	}, true, false)
+	err = p.ResolveServicesEnvironment(false)
+	assert.NilError(t, err)
+	service, err := p.GetService("Test")
 	assert.NilError(t, err)
-	assert.Equal(t, "YES", *s.Environment["HALLO"])
+	assert.Equal(t, "YES", *service.Environment["HALLO"])
 }
 
 func TestLoadServiceWithVolumes(t *testing.T) {

loader/normalize.go

@@ -85,6 +85,9 @@ func normalize(project *types.Project, resolvePaths bool) error {
 			}
 			s.Build.Args = s.Build.Args.Resolve(fn)
 		}
+		for j, f := range s.EnvFile {
+			s.EnvFile[j] = absPath(project.WorkingDir, f)
+		}
 		s.Environment = s.Environment.Resolve(fn)
 
 		err := relocateLogDriver(&s)

types/project.go

@@ -17,28 +17,32 @@
 package types
 
 import (
+	"bytes"
 	"fmt"
+	"github.com/pkg/errors"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
 
+	"github.com/compose-spec/compose-go/dotenv"
 	"github.com/distribution/distribution/v3/reference"
 	godigest "github.com/opencontainers/go-digest"
 	"golang.org/x/sync/errgroup"
 )
 
 // Project is the result of loading a set of compose files
 type Project struct {
-	Name         string            `yaml:"name,omitempty" json:"name,omitempty"`
-	WorkingDir   string            `yaml:"-" json:"-"`
-	Services     Services          `json:"services"`
-	Networks     Networks          `yaml:",omitempty" json:"networks,omitempty"`
-	Volumes      Volumes           `yaml:",omitempty" json:"volumes,omitempty"`
-	Secrets      Secrets           `yaml:",omitempty" json:"secrets,omitempty"`
-	Configs      Configs           `yaml:",omitempty" json:"configs,omitempty"`
-	Extensions   Extensions        `yaml:",inline" json:"-"` // https://github.com/golang/go/issues/6213
-	ComposeFiles []string          `yaml:"-" json:"-"`
-	Environment  map[string]string `yaml:"-" json:"-"`
+	Name         string     `yaml:"name,omitempty" json:"name,omitempty"`
+	WorkingDir   string     `yaml:"-" json:"-"`
+	Services     Services   `json:"services"`
+	Networks     Networks   `yaml:",omitempty" json:"networks,omitempty"`
+	Volumes      Volumes    `yaml:",omitempty" json:"volumes,omitempty"`
+	Secrets      Secrets    `yaml:",omitempty" json:"secrets,omitempty"`
+	Configs      Configs    `yaml:",omitempty" json:"configs,omitempty"`
+	Extensions   Extensions `yaml:",inline" json:"-"` // https://github.com/golang/go/issues/6213
+	ComposeFiles []string   `yaml:"-" json:"-"`
+	Environment  Mapping    `yaml:"-" json:"-"`
 
 	// DisabledServices track services which have been disable as profile is not active
 	DisabledServices Services `yaml:"-" json:"-"`
@@ -353,3 +357,49 @@ func (p *Project) ResolveImages(resolver func(named reference.Named) (godigest.D
 	}
 	return eg.Wait()
 }
+
+// ResolveServicesEnvironment parse env_files set for services to resolve the actual environment map for services
+func (p Project) ResolveServicesEnvironment(discardEnvFiles bool) error {
+	for i, service := range p.Services {
+		service.Environment = service.Environment.Resolve(p.Environment.Resolve)
+
+		environment := MappingWithEquals{}
+		// resolve variables based on other files we already parsed, + project's environment
+		var resolve dotenv.LookupFn = func(s string) (string, bool) {
+			v, ok := environment[s]
+			if ok && v != nil {
+				return *v, ok
+			}
+			return p.Environment.Resolve(s)
+		}
+
+		for _, envFile := range service.EnvFile {
+			file, err := os.Open(envFile)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to read %s", file)
+			}
+
+			b, err := io.ReadAll(file)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to read %s", file)
+			}
+
+			// Do not defer to avoid it inside a loop
+			file.Close() //nolint:errcheck
+
+			fileVars, err := dotenv.ParseWithLookup(bytes.NewBuffer(b), resolve)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to parse %s", file)
+			}
+			environment.OverrideBy(Mapping(fileVars).ToMappingWithEquals())
+		}
+
+		service.Environment = environment.OverrideBy(service.Environment)
+
+		if discardEnvFiles {
+			service.EnvFile = nil
+		}
+		p.Services[i] = service
+	}
+	return nil
+}

types/types.go

@@ -357,12 +357,12 @@ type ThrottleDevice struct {
 // ShellCommand is a string or list of string args.
 //
 // When marshaled to YAML, nil command fields will be omitted if `omitempty`
-// is specified as a struct tag. Explicitly empty commands (i.e. `[]` or `''`)
+// is specified as a struct tag. Explicitly empty commands (i.e. `[]` or `”`)
 // will serialize to an empty array (`[]`).
 //
 // When marshaled to JSON, the `omitempty` struct must NOT be specified.
 // If the command field is nil, it will be serialized as `null`.
-// Explicitly empty commands (i.e. `[]` or `''`) will serialize to an empty
+// Explicitly empty commands (i.e. `[]` or `”`) will serialize to an empty
 // array (`[]`).
 //
 // The distinction between nil and explicitly empty is important to distinguish
@@ -482,6 +482,21 @@ func NewMapping(values []string) Mapping {
 	return mapping
 }
 
+// ToMappingWithEquals converts Mapping into a MappingWithEquals with pointer references
+func (m Mapping) ToMappingWithEquals() MappingWithEquals {
+	mapping := MappingWithEquals{}
+	for k, v := range m {
+		v := v
+		mapping[k] = &v
+	}
+	return mapping
+}
+
+func (m Mapping) Resolve(s string) (string, bool) {
+	v, ok := m[s]
+	return v, ok
+}
+
 // Labels is a mapping type for labels
 type Labels map[string]string