diff --git a/.golangci.yml b/.golangci.yml index becfb8f2b93..f5e2325eabe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -131,6 +131,7 @@ linters: - lll - path: _test\.go linters: + - gosec - errcheck - funlen - maligned diff --git a/cmd/relayproxy/config/config.go b/cmd/relayproxy/config/config.go index f7e7255ba62..06bf61cfd0d 100644 --- a/cmd/relayproxy/config/config.go +++ b/cmd/relayproxy/config/config.go @@ -18,6 +18,7 @@ import ( "github.com/knadh/koanf/providers/posflag" "github.com/knadh/koanf/v2" "github.com/spf13/pflag" + ffclient "github.com/thomaspoignant/go-feature-flag" "github.com/xitongsys/parquet-go/parquet" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -45,6 +46,7 @@ var DefaultExporter = struct { MaxEventInMemory int64 ParquetCompressionCodec string LogLevel string + ExporterEventType ffclient.ExporterEventType }{ Format: "JSON", LogFormat: "[{{ .FormattedDate}}] user=\"{{ .UserKey}}\", flag=\"{{ .Key}}\", value=\"{{ .Value}}\"", @@ -55,6 +57,7 @@ var DefaultExporter = struct { MaxEventInMemory: 100000, ParquetCompressionCodec: parquet.CompressionCodec_SNAPPY.String(), LogLevel: DefaultLogLevel, + ExporterEventType: ffclient.FeatureEventExporter, } // New is reading the configuration file diff --git a/cmd/relayproxy/config/exporter.go b/cmd/relayproxy/config/exporter.go index 64073b86ec9..73efb539c84 100644 --- a/cmd/relayproxy/config/exporter.go +++ b/cmd/relayproxy/config/exporter.go @@ -33,6 +33,7 @@ type ExporterConf struct { AccountName string `mapstructure:"accountName" koanf:"accountname"` AccountKey string `mapstructure:"accountKey" koanf:"accountkey"` Container string `mapstructure:"container" koanf:"container"` + ExporterEventType string `mapstructure:"eventType" koanf:"eventtype"` } func (c *ExporterConf) IsValid() error { diff --git a/cmd/relayproxy/controller/collect_eval_data.go b/cmd/relayproxy/controller/collect_eval_data.go index 06a12710c5f..d9400bbf2a6 100644 --- a/cmd/relayproxy/controller/collect_eval_data.go +++ b/cmd/relayproxy/controller/collect_eval_data.go @@ -1,15 +1,18 @@ package controller import ( + "encoding/json" "fmt" "net/http" "strconv" + "github.com/go-viper/mapstructure/v2" "github.com/labstack/echo/v4" ffclient "github.com/thomaspoignant/go-feature-flag" "github.com/thomaspoignant/go-feature-flag/cmd/relayproxy/config" "github.com/thomaspoignant/go-feature-flag/cmd/relayproxy/metric" "github.com/thomaspoignant/go-feature-flag/cmd/relayproxy/model" + "github.com/thomaspoignant/go-feature-flag/exporter" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" @@ -63,28 +66,87 @@ func (h *collectEvalData) Handler(c echo.Context) error { _, span := tracer.Start(c.Request().Context(), "collectEventData") defer span.End() span.SetAttributes(attribute.Int("collectEventData.eventCollectionSize", len(reqBody.Events))) + counterTracking := 0 + counterEvaluation := 0 for _, event := range reqBody.Events { - if event.Source == "" { - event.Source = "PROVIDER_CACHE" + switch event["kind"] { + case "tracking": + e, err := convertTrackingEvent(event, h.logger) + if err != nil { + h.logger.Error( + "impossible to convert the event to a tracking event", + zap.Error(err), + ) + continue + } + h.goFF.CollectTrackingEventData(e) + counterTracking++ + default: + e, err := convertFeatureEvent(event, reqBody.Meta, h.logger) + if err != nil { + h.logger.Error("impossible to convert the event to a feature event", zap.Error(err)) + continue + } + h.goFF.CollectEventData(e) + counterEvaluation++ } - // force the creation date to be a unix timestamp - if event.CreationDate > 9999999999 { - h.logger.Warn( - "creationDate received is in milliseconds, we convert it to seconds", - zap.Int64("creationDate", event.CreationDate)) - // if we receive a timestamp in milliseconds, we convert it to seconds - // but since it is totally possible to have a timestamp in seconds that is bigger than 9999999999 - // we will accept timestamp up to 9999999999 (2286-11-20 18:46:39 +0100 CET) - event.CreationDate, _ = strconv.ParseInt( - strconv.FormatInt(event.CreationDate, 10)[:10], 10, 64) - } - if reqBody.Meta != nil { - event.Metadata = reqBody.Meta - } - h.goFF.CollectEventData(event) } + span.SetAttributes(attribute.Int("collectEventData.trackingCollectionSize", counterTracking)) + span.SetAttributes( + attribute.Int("collectEventData.evaluationCollectionSize", counterEvaluation), + ) h.metrics.IncCollectEvalData(float64(len(reqBody.Events))) return c.JSON(http.StatusOK, model.CollectEvalDataResponse{ IngestedContentCount: len(reqBody.Events), }) } + +func convertTrackingEvent( + event map[string]any, + logger *zap.Logger, +) (exporter.TrackingEvent, error) { + var e exporter.TrackingEvent + marshalled, err := json.Marshal(event) + if err != nil { + return exporter.TrackingEvent{}, err + } + err = json.Unmarshal(marshalled, &e) + if err != nil { + return exporter.TrackingEvent{}, err + } + e.CreationDate = formatCreationDate(e.CreationDate, logger) + return e, nil +} + +func convertFeatureEvent(event map[string]any, + metadata exporter.FeatureEventMetadata, + logger *zap.Logger) (exporter.FeatureEvent, error) { + var e exporter.FeatureEvent + err := mapstructure.Decode(event, &e) + if err != nil { + return exporter.FeatureEvent{}, err + } + if e.Source == "" { + e.Source = "PROVIDER_CACHE" + } + if metadata != nil { + e.Metadata = metadata + } + e.CreationDate = formatCreationDate(e.CreationDate, logger) + return e, nil +} + +func formatCreationDate(input int64, logger *zap.Logger) int64 { + if input > 9999999999 { + logger.Warn( + "creationDate received is in milliseconds, we convert it to seconds", + zap.Int64("creationDate", input)) + // if we receive a timestamp in milliseconds, we convert it to seconds + // but since it is totally possible to have a timestamp in seconds that is bigger than 9999999999 + // we will accept timestamp up to 9999999999 (2286-11-20 18:46:39 +0100 CET) + converted, _ := strconv.ParseInt( + strconv.FormatInt(input, 10)[:10], 10, 64) + return converted + } + return input +} diff --git a/cmd/relayproxy/controller/collect_eval_data_test.go b/cmd/relayproxy/controller/collect_eval_data_test.go index d31350e7363..0fe51e1fcc2 100644 --- a/cmd/relayproxy/controller/collect_eval_data_test.go +++ b/cmd/relayproxy/controller/collect_eval_data_test.go @@ -184,3 +184,62 @@ func Test_collect_eval_data_Handler(t *testing.T) { }) } } + +func Test_collect_tracking_and_evaluation_events(t *testing.T) { + evalExporter, err := os.CreateTemp("", "evalExport.json") + assert.NoError(t, err) + trackingExporter, err := os.CreateTemp("", "trackExport.json") + assert.NoError(t, err) + defer func() { + _ = os.Remove(evalExporter.Name()) + _ = os.Remove(trackingExporter.Name()) + }() + + goFF, _ := ffclient.New(ffclient.Config{ + PollingInterval: 10 * time.Second, + LeveledLogger: slog.Default(), + Context: context.Background(), + Retriever: &fileretriever.Retriever{ + Path: configFlagsLocation, + }, + + DataExporters: []ffclient.DataExporter{ + { + FlushInterval: 10 * time.Second, + MaxEventInMemory: 10000, + Exporter: &fileexporter.Exporter{Filename: evalExporter.Name()}, + }, + { + FlushInterval: 10 * time.Second, + MaxEventInMemory: 10000, + Exporter: &fileexporter.Exporter{Filename: trackingExporter.Name()}, + ExporterEventType: ffclient.TrackingEventExporter, + }, + }, + }) + logger, err := zap.NewDevelopment() + require.NoError(t, err) + ctrl := controller.NewCollectEvalData(goFF, metric.Metrics{}, logger) + + bodyReq, err := os.ReadFile( + "../testdata/controller/collect_eval_data/valid_request_mix_tracking_evaluation.json") + assert.NoError(t, err) + e := echo.New() + rec := httptest.NewRecorder() + + req := httptest.NewRequest(echo.POST, "/v1/data/collector", strings.NewReader(string(bodyReq))) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + c := e.NewContext(req, rec) + c.SetPath("/v1/data/collector") + handlerErr := ctrl.Handler(c) + assert.NoError(t, handlerErr) + goFF.Close() + evalEvents, err := os.ReadFile(evalExporter.Name()) + assert.NoError(t, err) + want := "{\"kind\":\"feature\",\"contextKind\":\"user\",\"userKey\":\"94a25909-20d8-40cc-8500-fee99b569345\",\"creationDate\":1680246000,\"key\":\"my-feature-flag\",\"variation\":\"admin-variation\",\"value\":\"string\",\"default\":false,\"version\":\"v1.0.0\",\"source\":\"PROVIDER_CACHE\",\"metadata\":{\"environment\":\"production\",\"sdkVersion\":\"v1.0.0\",\"source\":\"my-source\",\"timestamp\":1680246000}}\n" + assert.JSONEq(t, want, string(evalEvents), "Invalid exported data") + wantTracking := "{\"kind\":\"tracking\",\"contextKind\":\"user\",\"userKey\":\"94a25909-20d8-40cc-8500-fee99b569345\",\"creationDate\":1680246020,\"key\":\"my-feature-flag\",\"evaluationContext\":{\"admin\":true,\"name\":\"john doe\",\"targetingKey\":\"94a25909-20d8-40cc-8500-fee99b569345\"},\"trackingEventDetails\":{\"value\":\"string\",\"version\":\"v1.0.0\"}}\n" + trackingEvents, err := os.ReadFile(trackingExporter.Name()) + assert.NoError(t, err) + assert.JSONEq(t, wantTracking, string(trackingEvents), "Invalid exported data") +} diff --git a/cmd/relayproxy/docs/docs.go b/cmd/relayproxy/docs/docs.go index 56f57a22906..e418fddfa2e 100644 --- a/cmd/relayproxy/docs/docs.go +++ b/cmd/relayproxy/docs/docs.go @@ -623,67 +623,6 @@ const docTemplate = `{ } } }, - "exporter.FeatureEvent": { - "type": "object", - "properties": { - "contextKind": { - "description": "ContextKind is the kind of context which generated an event. This will only be \"anonymousUser\" for events generated\non behalf of an anonymous user or the reserved word \"user\" for events generated on behalf of a non-anonymous user", - "type": "string", - "example": "user" - }, - "creationDate": { - "description": "CreationDate When the feature flag was requested at Unix epoch time in milliseconds.", - "type": "integer", - "example": 1680246000011 - }, - "default": { - "description": "Default value is set to true if feature flag evaluation failed, in which case the value returned was the default\nvalue passed to variation. If the default field is omitted, it is assumed to be false.", - "type": "boolean", - "example": false - }, - "key": { - "description": "Key of the feature flag requested.", - "type": "string", - "example": "my-feature-flag" - }, - "kind": { - "description": "Kind for a feature event is feature.\nA feature event will only be generated if the trackEvents attribute of the flag is set to true.", - "type": "string", - "example": "feature" - }, - "metadata": { - "description": "Metadata are static information added in the providers to give context about the events generated.", - "allOf": [ - { - "$ref": "#/definitions/exporter.FeatureEventMetadata" - } - ] - }, - "source": { - "description": "Source indicates where the event was generated.\nThis is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.", - "type": "string", - "example": "SERVER" - }, - "userKey": { - "description": "UserKey The key of the user object used in a feature flag evaluation. Details for the user object used in a feature\nflag evaluation as reported by the \"feature\" event are transmitted periodically with a separate index event.", - "type": "string", - "example": "94a25909-20d8-40cc-8500-fee99b569345" - }, - "value": { - "description": "Value of the feature flag returned by feature flag evaluation." - }, - "variation": { - "description": "Variation of the flag requested. Flag variation values can be \"True\", \"False\", \"Default\" or \"SdkDefault\"\ndepending on which value was taken during flag evaluation. \"SdkDefault\" is used when an error is detected and the\ndefault value passed during the call to your variation is used.", - "type": "string", - "example": "admin-variation" - }, - "version": { - "description": "Version contains the version of the flag. If the field is omitted for the flag in the configuration file\nthe default version will be 0.", - "type": "string", - "example": "v1.0.0" - } - } - }, "exporter.FeatureEventMetadata": { "type": "object", "additionalProperties": true @@ -748,10 +687,11 @@ const docTemplate = `{ "type": "object", "properties": { "events": { - "description": "Events is the list of the event we send in the payload", + "description": "Events is the list of the event we send in the payload\nhere the type is any because we will unmarshal later in the different event types", "type": "array", "items": { - "$ref": "#/definitions/exporter.FeatureEvent" + "type": "object", + "additionalProperties": {} } }, "meta": { diff --git a/cmd/relayproxy/docs/swagger.json b/cmd/relayproxy/docs/swagger.json index 94ca6026b32..80c7f4310c2 100644 --- a/cmd/relayproxy/docs/swagger.json +++ b/cmd/relayproxy/docs/swagger.json @@ -615,67 +615,6 @@ } } }, - "exporter.FeatureEvent": { - "type": "object", - "properties": { - "contextKind": { - "description": "ContextKind is the kind of context which generated an event. This will only be \"anonymousUser\" for events generated\non behalf of an anonymous user or the reserved word \"user\" for events generated on behalf of a non-anonymous user", - "type": "string", - "example": "user" - }, - "creationDate": { - "description": "CreationDate When the feature flag was requested at Unix epoch time in milliseconds.", - "type": "integer", - "example": 1680246000011 - }, - "default": { - "description": "Default value is set to true if feature flag evaluation failed, in which case the value returned was the default\nvalue passed to variation. If the default field is omitted, it is assumed to be false.", - "type": "boolean", - "example": false - }, - "key": { - "description": "Key of the feature flag requested.", - "type": "string", - "example": "my-feature-flag" - }, - "kind": { - "description": "Kind for a feature event is feature.\nA feature event will only be generated if the trackEvents attribute of the flag is set to true.", - "type": "string", - "example": "feature" - }, - "metadata": { - "description": "Metadata are static information added in the providers to give context about the events generated.", - "allOf": [ - { - "$ref": "#/definitions/exporter.FeatureEventMetadata" - } - ] - }, - "source": { - "description": "Source indicates where the event was generated.\nThis is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.", - "type": "string", - "example": "SERVER" - }, - "userKey": { - "description": "UserKey The key of the user object used in a feature flag evaluation. Details for the user object used in a feature\nflag evaluation as reported by the \"feature\" event are transmitted periodically with a separate index event.", - "type": "string", - "example": "94a25909-20d8-40cc-8500-fee99b569345" - }, - "value": { - "description": "Value of the feature flag returned by feature flag evaluation." - }, - "variation": { - "description": "Variation of the flag requested. Flag variation values can be \"True\", \"False\", \"Default\" or \"SdkDefault\"\ndepending on which value was taken during flag evaluation. \"SdkDefault\" is used when an error is detected and the\ndefault value passed during the call to your variation is used.", - "type": "string", - "example": "admin-variation" - }, - "version": { - "description": "Version contains the version of the flag. If the field is omitted for the flag in the configuration file\nthe default version will be 0.", - "type": "string", - "example": "v1.0.0" - } - } - }, "exporter.FeatureEventMetadata": { "type": "object", "additionalProperties": true @@ -740,10 +679,11 @@ "type": "object", "properties": { "events": { - "description": "Events is the list of the event we send in the payload", + "description": "Events is the list of the event we send in the payload\nhere the type is any because we will unmarshal later in the different event types", "type": "array", "items": { - "$ref": "#/definitions/exporter.FeatureEvent" + "type": "object", + "additionalProperties": {} } }, "meta": { diff --git a/cmd/relayproxy/docs/swagger.yaml b/cmd/relayproxy/docs/swagger.yaml index 174385bedbc..aec9c758747 100644 --- a/cmd/relayproxy/docs/swagger.yaml +++ b/cmd/relayproxy/docs/swagger.yaml @@ -14,68 +14,6 @@ definitions: refreshed: type: boolean type: object - exporter.FeatureEvent: - properties: - contextKind: - description: |- - ContextKind is the kind of context which generated an event. This will only be "anonymousUser" for events generated - on behalf of an anonymous user or the reserved word "user" for events generated on behalf of a non-anonymous user - example: user - type: string - creationDate: - description: CreationDate When the feature flag was requested at Unix epoch - time in milliseconds. - example: 1680246000011 - type: integer - default: - description: |- - Default value is set to true if feature flag evaluation failed, in which case the value returned was the default - value passed to variation. If the default field is omitted, it is assumed to be false. - example: false - type: boolean - key: - description: Key of the feature flag requested. - example: my-feature-flag - type: string - kind: - description: |- - Kind for a feature event is feature. - A feature event will only be generated if the trackEvents attribute of the flag is set to true. - example: feature - type: string - metadata: - allOf: - - $ref: '#/definitions/exporter.FeatureEventMetadata' - description: Metadata are static information added in the providers to give - context about the events generated. - source: - description: |- - Source indicates where the event was generated. - This is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache. - example: SERVER - type: string - userKey: - description: |- - UserKey The key of the user object used in a feature flag evaluation. Details for the user object used in a feature - flag evaluation as reported by the "feature" event are transmitted periodically with a separate index event. - example: 94a25909-20d8-40cc-8500-fee99b569345 - type: string - value: - description: Value of the feature flag returned by feature flag evaluation. - variation: - description: |- - Variation of the flag requested. Flag variation values can be "True", "False", "Default" or "SdkDefault" - depending on which value was taken during flag evaluation. "SdkDefault" is used when an error is detected and the - default value passed during the call to your variation is used. - example: admin-variation - type: string - version: - description: |- - Version contains the version of the flag. If the field is omitted for the flag in the configuration file - the default version will be 0. - example: v1.0.0 - type: string - type: object exporter.FeatureEventMetadata: additionalProperties: true type: object @@ -124,9 +62,12 @@ definitions: model.CollectEvalDataRequest: properties: events: - description: Events is the list of the event we send in the payload + description: |- + Events is the list of the event we send in the payload + here the type is any because we will unmarshal later in the different event types items: - $ref: '#/definitions/exporter.FeatureEvent' + additionalProperties: {} + type: object type: array meta: allOf: diff --git a/cmd/relayproxy/model/collect_eval_data_request.go b/cmd/relayproxy/model/collect_eval_data_request.go index 9891aa7503e..dd17f2039eb 100644 --- a/cmd/relayproxy/model/collect_eval_data_request.go +++ b/cmd/relayproxy/model/collect_eval_data_request.go @@ -10,5 +10,6 @@ type CollectEvalDataRequest struct { Meta exporter.FeatureEventMetadata `json:"meta"` // Events is the list of the event we send in the payload - Events []exporter.FeatureEvent `json:"events"` + // here the type is any because we will unmarshal later in the different event types + Events []map[string]any `json:"events"` } diff --git a/cmd/relayproxy/service/gofeatureflag.go b/cmd/relayproxy/service/gofeatureflag.go index ecc46fde3bb..e6c49ad5dcc 100644 --- a/cmd/relayproxy/service/gofeatureflag.go +++ b/cmd/relayproxy/service/gofeatureflag.go @@ -168,6 +168,10 @@ func initDataExporters(proxyConf *config.Config) ([]ffclient.DataExporter, error } func initDataExporter(c *config.ExporterConf) (ffclient.DataExporter, error) { + exporterEventType := c.ExporterEventType + if exporterEventType == "" { + exporterEventType = config.DefaultExporter.ExporterEventType + } dataExp := ffclient.DataExporter{ FlushInterval: func() time.Duration { if c.FlushInterval != 0 { @@ -181,6 +185,7 @@ func initDataExporter(c *config.ExporterConf) (ffclient.DataExporter, error) { } return config.DefaultExporter.MaxEventInMemory }(), + ExporterEventType: exporterEventType, } var err error diff --git a/cmd/relayproxy/service/gofeatureflag_test.go b/cmd/relayproxy/service/gofeatureflag_test.go index 4209ef4f3d5..4ce40f7a3f4 100644 --- a/cmd/relayproxy/service/gofeatureflag_test.go +++ b/cmd/relayproxy/service/gofeatureflag_test.go @@ -477,6 +477,7 @@ func Test_initExporter(t *testing.T) { Secret: "1234", Meta: nil, }, + ExporterEventType: ffclient.FeatureEventExporter, }, wantType: &webhookexporter.Exporter{}, }, @@ -498,6 +499,7 @@ func Test_initExporter(t *testing.T) { CsvTemplate: config.DefaultExporter.CsvFormat, ParquetCompressionCodec: parquet.CompressionCodec_UNCOMPRESSED.String(), }, + ExporterEventType: ffclient.FeatureEventExporter, }, wantType: &fileexporter.Exporter{}, }, @@ -513,6 +515,7 @@ func Test_initExporter(t *testing.T) { Exporter: &logsexporter.Exporter{ LogFormat: config.DefaultExporter.LogFormat, }, + ExporterEventType: ffclient.FeatureEventExporter, }, wantType: &logsexporter.Exporter{}, }, @@ -595,6 +598,7 @@ func Test_initExporter(t *testing.T) { CsvTemplate: config.DefaultExporter.CsvFormat, ParquetCompressionCodec: config.DefaultExporter.ParquetCompressionCodec, }, + ExporterEventType: ffclient.FeatureEventExporter, }, wantType: &gcstorageexporter.Exporter{}, }, @@ -608,6 +612,7 @@ func Test_initExporter(t *testing.T) { Topic: "example-topic", Addresses: []string{"addr1", "addr2"}, }, + ExporterEventType: ffclient.FeatureEventExporter, }, want: ffclient.DataExporter{ FlushInterval: config.DefaultExporter.FlushInterval, @@ -619,6 +624,7 @@ func Test_initExporter(t *testing.T) { Addresses: []string{"addr1", "addr2"}, }, }, + ExporterEventType: ffclient.FeatureEventExporter, }, wantType: &kafkaexporter.Exporter{}, }, @@ -662,6 +668,7 @@ func Test_initExporter(t *testing.T) { CsvTemplate: config.DefaultExporter.CsvFormat, ParquetCompressionCodec: config.DefaultExporter.ParquetCompressionCodec, }, + ExporterEventType: ffclient.FeatureEventExporter, }, wantType: &azureexporter.Exporter{}, }, diff --git a/cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_mix_tracking_evaluation.json b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_mix_tracking_evaluation.json new file mode 100644 index 00000000000..14d9305f900 --- /dev/null +++ b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_mix_tracking_evaluation.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "contextKind": "user", + "creationDate": 1680246000, + "default": false, + "key": "my-feature-flag", + "kind": "feature", + "userKey": "94a25909-20d8-40cc-8500-fee99b569345", + "value": "string", + "variation": "admin-variation", + "version": "v1.0.0" + }, + { + "kind": "tracking", + "creationDate": 1680246020, + "contextKind": "user", + "userKey": "94a25909-20d8-40cc-8500-fee99b569345", + "key": "my-feature-flag", + "evaluationContext": { + "targetingKey": "94a25909-20d8-40cc-8500-fee99b569345", + "name": "john doe", + "admin": true + }, + "trackingEventDetails": { + "value": "string", + "version": "v1.0.0" + } + } + ], + "meta": { + "environment": "production", + "sdkVersion": "v1.0.0", + "source": "my-source", + "timestamp": 1680246000 + } +} \ No newline at end of file diff --git a/config_exporter.go b/config_exporter.go index c54c7f69cbe..27fee1d0862 100644 --- a/config_exporter.go +++ b/config_exporter.go @@ -6,6 +6,13 @@ import ( "github.com/thomaspoignant/go-feature-flag/exporter" ) +type ExporterEventType = string + +const ( + TrackingEventExporter ExporterEventType = "tracking" + FeatureEventExporter ExporterEventType = "feature" +) + // DataExporter is the configuration of your export target. type DataExporter struct { // FlushInterval is the interval we are waiting to export the data. @@ -22,4 +29,8 @@ type DataExporter struct { // Exporter is the configuration of your exporter. // You can see all available exporter in the exporter package. Exporter exporter.CommonExporter + + // ExporterEventType is the type of event the exporter is expecting. + // The default type if not set is FeatureEventExporter. + ExporterEventType ExporterEventType } diff --git a/exporter/azureexporter/exporter.go b/exporter/azureexporter/exporter.go index eb094ba4bff..cf6f928f21e 100644 --- a/exporter/azureexporter/exporter.go +++ b/exporter/azureexporter/exporter.go @@ -55,7 +55,7 @@ func (f *Exporter) initializeAzureClient() (*azblob.Client, error) { func (f *Exporter) Export( ctx context.Context, logger *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + featureEvents []exporter.ExportableEvent, ) error { if f.AccountName == "" { return fmt.Errorf("you should specify an AccountName. %v is invalid", f.AccountName) diff --git a/exporter/azureexporter/exporter_test.go b/exporter/azureexporter/exporter_test.go index f351eb0d8ea..e881067f145 100644 --- a/exporter/azureexporter/exporter_test.go +++ b/exporter/azureexporter/exporter_test.go @@ -26,7 +26,7 @@ func TestAzureBlobStorage_Export(t *testing.T) { tests := []struct { name string exporter azureexporter.Exporter - events []exporter.FeatureEvent + events []exporter.ExportableEvent wantErr assert.ErrorAssertionFunc wantBlobName string }{ @@ -37,8 +37,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -54,8 +54,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -71,8 +71,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -88,8 +88,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -105,8 +105,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -122,8 +122,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -137,8 +137,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountName: azurite.AccountName, AccountKey: azurite.AccountKey, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -148,8 +148,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { { name: "Should error with nil container", exporter: azureexporter.Exporter{}, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -161,8 +161,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { exporter: azureexporter.Exporter{ AccountName: "", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -176,8 +176,8 @@ func TestAzureBlobStorage_Export(t *testing.T) { AccountKey: azurite.AccountKey, Container: containerName, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, diff --git a/exporter/common.go b/exporter/common.go index 152173f2c89..62315fd3f33 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -2,7 +2,6 @@ package exporter import ( "bytes" - "encoding/json" "os" "strconv" "strings" @@ -44,15 +43,3 @@ func ComputeFilename(template *template.Template, format string) (string, error) }) return buf.String(), err } - -func FormatEventInCSV(csvTemplate *template.Template, event FeatureEvent) ([]byte, error) { - var buf bytes.Buffer - err := csvTemplate.Execute(&buf, event) - return buf.Bytes(), err -} - -func FormatEventInJSON(event FeatureEvent) ([]byte, error) { - b, err := json.Marshal(event) - b = append(b, []byte("\n")...) - return b, err -} diff --git a/exporter/common_test.go b/exporter/common_test.go index 5f4ef766462..dc21411c069 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -171,11 +171,11 @@ func TestFormatEventInCSV(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := exporter.FormatEventInCSV(tt.args.csvTemplate, tt.args.event) + got, err := tt.args.event.FormatInCSV(tt.args.csvTemplate) if !tt.wantErr( t, err, - fmt.Sprintf("FormatEventInCSV(%v, %v)", tt.args.csvTemplate, tt.args.event), + fmt.Sprintf("FormatInCSV(%v, %v)", tt.args.csvTemplate, tt.args.event), ) { return } @@ -183,7 +183,7 @@ func TestFormatEventInCSV(t *testing.T) { t, tt.want, string(got), - "FormatEventInCSV(%v, %v)", + "FormatInCSV(%v, %v)", tt.args.csvTemplate, tt.args.event, ) @@ -213,11 +213,11 @@ func TestFormatEventInJSON(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := exporter.FormatEventInJSON(tt.args.event) - if !tt.wantErr(t, err, fmt.Sprintf("FormatEventInJSON(%v)", tt.args.event)) { + got, err := tt.args.event.FormatInJSON() + if !tt.wantErr(t, err, fmt.Sprintf("FormatInJSON(%v)", tt.args.event)) { return } - assert.Equalf(t, tt.want, string(got), "FormatEventInJSON(%v)", tt.args.event) + assert.Equalf(t, tt.want, string(got), "FormatInJSON(%v)", tt.args.event) }) } } diff --git a/exporter/data_exporter.go b/exporter/data_exporter.go index 086f2e1b7af..d175c7d9ab5 100644 --- a/exporter/data_exporter.go +++ b/exporter/data_exporter.go @@ -15,7 +15,7 @@ const ( defaultMaxEventInMemory = int64(100000) ) -type DataExporter[T any] interface { +type DataExporter[T ExportableEvent] interface { // Start is launching the ticker to periodically flush the data Start() // Stop is stopping the ticker @@ -35,7 +35,7 @@ type Config struct { MaxEventInMemory int64 } -type dataExporterImpl[T any] struct { +type dataExporterImpl[T ExportableEvent] struct { ctx context.Context consumerID string eventStore *EventStore[T] @@ -48,7 +48,7 @@ type dataExporterImpl[T any] struct { // NewDataExporter create a new DataExporter with the given exporter and his consumer information to consume the data // from the shared event store. -func NewDataExporter[T any](ctx context.Context, exporter Config, consumerID string, +func NewDataExporter[T ExportableEvent](ctx context.Context, exporter Config, consumerID string, eventStore *EventStore[T], logger *fflog.FFLogger) DataExporter[T] { if ctx == nil { ctx = context.Background() @@ -135,14 +135,14 @@ func (d *dataExporterImpl[T]) sendEvents(ctx context.Context, events []T) error return nil } switch exp := d.exporter.Exporter.(type) { - case DeprecatedExporter: + case DeprecatedExporterV1: var legacyLogger *log.Logger if d.logger != nil { legacyLogger = d.logger.GetLogLogger(slog.LevelError) } switch events := any(events).(type) { case []FeatureEvent: - // use dc exporter as a DeprecatedExporter + // use dc exporter as a DeprecatedExporterV1 err := exp.Export(ctx, legacyLogger, events) slog.Warn("You are using an exporter with the old logger."+ "Please update your custom exporter to comply to the new Exporter interface.", @@ -153,7 +153,7 @@ func (d *dataExporterImpl[T]) sendEvents(ctx context.Context, events []T) error default: return fmt.Errorf("trying to send unknown object to the exporter (deprecated)") } - case Exporter: + case DeprecatedExporterV2: switch events := any(events).(type) { case []FeatureEvent: err := exp.Export(ctx, d.logger, events) @@ -163,6 +163,15 @@ func (d *dataExporterImpl[T]) sendEvents(ctx context.Context, events []T) error default: return fmt.Errorf("trying to send unknown object to the exporter") } + case Exporter: + exportableEvents := make([]ExportableEvent, len(events)) + for i, event := range events { + exportableEvents[i] = ExportableEvent(event) + } + err := exp.Export(ctx, d.logger, exportableEvents) + if err != nil { + return fmt.Errorf("error while exporting data: %w", err) + } default: return fmt.Errorf("this is not a valid exporter") } diff --git a/exporter/data_exporter_test.go b/exporter/data_exporter_test.go index 4d67c7c0234..ed6b995e0c8 100644 --- a/exporter/data_exporter_test.go +++ b/exporter/data_exporter_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/thomaspoignant/go-feature-flag/exporter" + "github.com/thomaspoignant/go-feature-flag/testutils" "github.com/thomaspoignant/go-feature-flag/testutils/mock" "github.com/thomaspoignant/go-feature-flag/testutils/slogutil" "github.com/thomaspoignant/go-feature-flag/utils/fflog" @@ -47,11 +48,6 @@ func TestDataExporterFlush_TriggerErrorIfNotKnowType(t *testing.T) { exporter mock.ExporterMock expectedLog string }{ - { - name: "classic exporter", - exporter: &mock.Exporter{}, - expectedLog: "trying to send unknown object to the exporter\n", - }, { name: "deprecated exporter", exporter: &mock.ExporterDeprecated{}, @@ -61,9 +57,9 @@ func TestDataExporterFlush_TriggerErrorIfNotKnowType(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - evStore := mock.NewEventStore[string]() + evStore := mock.NewEventStore[testutils.ExportableMockEvent]() for i := 0; i < 100; i++ { - evStore.Add("feature") + evStore.Add(testutils.NewExportableMockEvent("feature")) } logFile, _ := os.CreateTemp("", "") @@ -72,11 +68,17 @@ func TestDataExporterFlush_TriggerErrorIfNotKnowType(t *testing.T) { defer func() { _ = os.Remove(logFile.Name()) }() exporterMock := tt.exporter - exp := exporter.NewDataExporter[string](context.TODO(), exporter.Config{ - Exporter: exporterMock, - FlushInterval: 0, - MaxEventInMemory: 0, - }, "id-consumer", &evStore, logger) + exp := exporter.NewDataExporter[testutils.ExportableMockEvent]( + context.TODO(), + exporter.Config{ + Exporter: exporterMock, + FlushInterval: 0, + MaxEventInMemory: 0, + }, + "id-consumer", + &evStore, + logger, + ) exp.Flush() // flush should error and not return any event diff --git a/exporter/even_store_test.go b/exporter/even_store_test.go index dd1370263fa..2b30061567f 100644 --- a/exporter/even_store_test.go +++ b/exporter/even_store_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/thomaspoignant/go-feature-flag/exporter" + "github.com/thomaspoignant/go-feature-flag/testutils" ) const defaultTestCleanQueueDuration = 100 * time.Millisecond @@ -18,7 +19,9 @@ func Test_ConsumerNameInvalid(t *testing.T) { t.Run( "GetPendingEventCount: should return an error if the consumer name is invalid", func(t *testing.T) { - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) eventStore.AddConsumer("consumer1") defer eventStore.Stop() _, err := eventStore.GetPendingEventCount("wrong name") @@ -28,12 +31,15 @@ func Test_ConsumerNameInvalid(t *testing.T) { t.Run( "ProcessPendingEvents: should return an error if the consumer name is invalid", func(t *testing.T) { - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) eventStore.AddConsumer("consumer1") defer eventStore.Stop() err := eventStore.ProcessPendingEvents( "wrong name", - func(ctx context.Context, events []string) error { return nil }) + func(ctx context.Context, events []testutils.ExportableMockEvent) error { return nil }, + ) assert.NotNil(t, err) }, ) @@ -41,7 +47,9 @@ func Test_ConsumerNameInvalid(t *testing.T) { func Test_SingleConsumer(t *testing.T) { consumerName := "consumer1" - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) eventStore.AddConsumer(consumerName) defer eventStore.Stop() got, _ := eventStore.GetPendingEventCount(consumerName) @@ -57,13 +65,11 @@ func Test_SingleConsumer(t *testing.T) { cancel() // stop producing // Consume - err := eventStore.ProcessPendingEvents( - consumerName, - func(ctx context.Context, events []string) error { + err := eventStore.ProcessPendingEvents(consumerName, + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 100, len(events)) return nil - }, - ) + }) assert.Nil(t, err) got, _ = eventStore.GetPendingEventCount(consumerName) assert.Equal(t, int64(0), got) @@ -76,13 +82,11 @@ func Test_SingleConsumer(t *testing.T) { got, _ = eventStore.GetPendingEventCount(consumerName) assert.Equal(t, int64(91), got) - err = eventStore.ProcessPendingEvents( - consumerName, - func(ctx context.Context, events []string) error { + err = eventStore.ProcessPendingEvents(consumerName, + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 91, len(events)) return nil - }, - ) + }) assert.Nil(t, err) time.Sleep(120 * time.Millisecond) // to wait until garbage collector remove the events @@ -91,7 +95,9 @@ func Test_SingleConsumer(t *testing.T) { func Test_MultipleConsumersSingleThread(t *testing.T) { consumerNames := []string{"consumer1", "consumer2"} - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) for _, name := range consumerNames { eventStore.AddConsumer(name) } @@ -107,13 +113,11 @@ func Test_MultipleConsumersSingleThread(t *testing.T) { consumer1Size, err := eventStore.GetPendingEventCount(consumerNames[0]) assert.Nil(t, err) assert.Equal(t, int64(1000), consumer1Size) - err = eventStore.ProcessPendingEvents( - consumerNames[0], - func(ctx context.Context, events []string) error { + err = eventStore.ProcessPendingEvents(consumerNames[0], + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 1000, len(events)) return nil - }, - ) + }) assert.Nil(t, err) // Produce a second time @@ -132,22 +136,18 @@ func Test_MultipleConsumersSingleThread(t *testing.T) { assert.Equal(t, int64(2000), consumer2Size) // Consumer with Consumer1 and Consumer2 - err = eventStore.ProcessPendingEvents( - consumerNames[0], - func(ctx context.Context, events []string) error { + err = eventStore.ProcessPendingEvents(consumerNames[0], + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 1000, len(events)) return nil - }, - ) + }) assert.Nil(t, err) - err = eventStore.ProcessPendingEvents( - consumerNames[1], - func(ctx context.Context, events []string) error { + err = eventStore.ProcessPendingEvents(consumerNames[1], + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 2000, len(events)) return nil - }, - ) + }) assert.Nil(t, err) // Check garbage collector @@ -157,7 +157,9 @@ func Test_MultipleConsumersSingleThread(t *testing.T) { func Test_MultipleConsumersMultipleGORoutines(t *testing.T) { consumerNames := []string{"consumer1", "consumer2"} - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) for _, name := range consumerNames { eventStore.AddConsumer(name) } @@ -169,40 +171,43 @@ func Test_MultipleConsumersMultipleGORoutines(t *testing.T) { time.Sleep(50 * time.Millisecond) wg := &sync.WaitGroup{} - consumFunc := func(eventStore exporter.EventStore[string], consumerName string) { + consumeFunc := func(eventStore exporter.EventStore[testutils.ExportableMockEvent], consumerName string, eventCounters *map[string]int) { defer wg.Done() - - err := eventStore.ProcessPendingEvents( - consumerName, - func(ctx context.Context, events []string) error { + err := eventStore.ProcessPendingEvents(consumerName, + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.True(t, len(events) > 0) return nil - }, - ) + }) assert.Nil(t, err) time.Sleep( 50 * time.Millisecond, ) // we wait to be sure that the producer has produce new events - err = eventStore.ProcessPendingEvents( - consumerName, - func(ctx context.Context, events []string) error { - assert.True(t, len(events) > 0) + err = eventStore.ProcessPendingEvents(consumerName, + func(ctx context.Context, events []testutils.ExportableMockEvent) error { + if eventCounters != nil { + (*eventCounters)[consumerName] = len(events) + } return nil - }, - ) + }) assert.Nil(t, err) } wg.Add(2) - go consumFunc(eventStore, consumerNames[0]) - go consumFunc(eventStore, consumerNames[1]) + eventCounters := map[string]int{} + go consumeFunc(eventStore, consumerNames[0], &eventCounters) + go consumeFunc(eventStore, consumerNames[1], &eventCounters) wg.Wait() + + assert.Greater(t, eventCounters[consumerNames[0]], 0) + assert.Greater(t, eventCounters[consumerNames[1]], 0) } func Test_ProcessPendingEventInError(t *testing.T) { consumerName := "consumer1" - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) eventStore.AddConsumer(consumerName) defer eventStore.Stop() // start producer @@ -216,7 +221,7 @@ func Test_ProcessPendingEventInError(t *testing.T) { // process is in error, so we are not able to update the offset err = eventStore.ProcessPendingEvents( consumerName, - func(ctx context.Context, events []string) error { + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 1000, len(events)) return fmt.Errorf("error") }, @@ -231,14 +236,14 @@ func Test_ProcessPendingEventInError(t *testing.T) { // process is not in error anymore err = eventStore.ProcessPendingEvents( consumerName, - func(ctx context.Context, events []string) error { + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 1000, len(events)) return nil }, ) assert.Nil(t, err) - // we have consume all the items + // we have consumed all the items consumer1Size, err = eventStore.GetPendingEventCount(consumerName) assert.Equal(t, 0, int(consumer1Size)) assert.Nil(t, err) @@ -246,7 +251,9 @@ func Test_ProcessPendingEventInError(t *testing.T) { func Test_WaitForEmptyClean(t *testing.T) { consumerNames := []string{"consumer1"} - eventStore := exporter.NewEventStore[string](defaultTestCleanQueueDuration) + eventStore := exporter.NewEventStore[testutils.ExportableMockEvent]( + defaultTestCleanQueueDuration, + ) for _, name := range consumerNames { eventStore.AddConsumer(name) } @@ -257,7 +264,7 @@ func Test_WaitForEmptyClean(t *testing.T) { startEventProducer(ctx, eventStore, 100, false) err := eventStore.ProcessPendingEvents( consumerNames[0], - func(ctx context.Context, events []string) error { + func(ctx context.Context, events []testutils.ExportableMockEvent) error { assert.Equal(t, 100, len(events)) return nil }, @@ -270,7 +277,7 @@ func Test_WaitForEmptyClean(t *testing.T) { func startEventProducer( ctx context.Context, - eventStore exporter.EventStore[string], + eventStore exporter.EventStore[testutils.ExportableMockEvent], produceMax int, randomizeProducingTime bool, ) { @@ -284,7 +291,7 @@ func startEventProducer( randomNumber := rand.Intn(10) + 1 time.Sleep(time.Duration(randomNumber) * time.Millisecond) } - eventStore.Add("Hello") + eventStore.Add(testutils.NewExportableMockEvent("Hello")) } } } diff --git a/exporter/event_store.go b/exporter/event_store.go index 250818bae98..707d5305fa0 100644 --- a/exporter/event_store.go +++ b/exporter/event_store.go @@ -10,9 +10,9 @@ import ( const minOffset = int64(math.MinInt64) -type eventStoreImpl[T any] struct { +type eventStoreImpl[T ExportableEvent] struct { // events is a list of events to store - events []Event[T] + events []EventStoreItem[T] // mutex to protect the events and consumers mutex sync.RWMutex // consumers is a map of consumers with their name as key @@ -25,9 +25,9 @@ type eventStoreImpl[T any] struct { cleanQueueInterval time.Duration } -func NewEventStore[T any](cleanQueueInterval time.Duration) EventStore[T] { +func NewEventStore[T ExportableEvent](cleanQueueInterval time.Duration) EventStore[T] { store := &eventStoreImpl[T]{ - events: make([]Event[T], 0), + events: make([]EventStoreItem[T], 0), mutex: sync.RWMutex{}, lastOffset: minOffset, stopPeriodicCleaning: make(chan struct{}), @@ -38,7 +38,7 @@ func NewEventStore[T any](cleanQueueInterval time.Duration) EventStore[T] { return store } -type EventList[T any] struct { +type EventList[T ExportableEvent] struct { Events []T InitialOffset int64 NewOffset int64 @@ -46,7 +46,7 @@ type EventList[T any] struct { // EventStore is the interface to store events and consume them. // It is a simple implementation of a queue with offsets. -type EventStore[T any] interface { +type EventStore[T ExportableEvent] interface { // AddConsumer is adding a new consumer to the Event store. // note that you can't add a consumer after the Event store has been started. AddConsumer(consumerID string) @@ -71,7 +71,7 @@ type EventStore[T any] interface { Stop() } -type Event[T any] struct { +type EventStoreItem[T ExportableEvent] struct { Offset int64 Data T } @@ -131,7 +131,7 @@ func (e *eventStoreImpl[T]) Add(data T) { e.mutex.Lock() defer e.mutex.Unlock() e.lastOffset++ - e.events = append(e.events, Event[T]{Offset: e.lastOffset, Data: data}) + e.events = append(e.events, EventStoreItem[T]{Offset: e.lastOffset, Data: data}) } // fetchPendingEvents is returning all the available item in the Event store for this consumer. diff --git a/exporter/exportable_event.go b/exporter/exportable_event.go new file mode 100644 index 00000000000..89e60c33b80 --- /dev/null +++ b/exporter/exportable_event.go @@ -0,0 +1,18 @@ +package exporter + +import ( + "text/template" +) + +type ExportableEvent interface { + // GetUserKey returns the unique key for the event. + GetUserKey() string + // GetKey returns the unique key for the event. + GetKey() string + // GetCreationDate returns the creationDate of the event. + GetCreationDate() int64 + // FormatInCSV FormatEventInCSV returns the event in CSV format. + FormatInCSV(csvTemplate *template.Template) ([]byte, error) + // FormatInJSON FormatEventInJSON returns the event in JSON format. + FormatInJSON() ([]byte, error) +} diff --git a/exporter/exporter.go b/exporter/exporter.go index 02690e45611..51e371a3cfa 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -7,19 +7,23 @@ import ( "github.com/thomaspoignant/go-feature-flag/utils/fflog" ) -// DeprecatedExporter is an interface to describe how an exporter looks like. +// DeprecatedExporterV1 is an interface to describe how an exporter looks like. // Deprecated: use Exporter instead. -type DeprecatedExporter interface { +type DeprecatedExporterV1 interface { CommonExporter // Export will send the data to the exporter. Export(context.Context, *log.Logger, []FeatureEvent) error } -type Exporter interface { +type DeprecatedExporterV2 interface { CommonExporter Export(context.Context, *fflog.FFLogger, []FeatureEvent) error } +type Exporter interface { + CommonExporter + Export(context.Context, *fflog.FFLogger, []ExportableEvent) error +} type CommonExporter interface { // IsBulk return false if we should directly send the data as soon as it is produce // and true if we collect the data to send them in bulk. diff --git a/exporter/feature_event.go b/exporter/feature_event.go index 61ad2bcd736..d7c16b132c3 100644 --- a/exporter/feature_event.go +++ b/exporter/feature_event.go @@ -1,7 +1,10 @@ package exporter import ( + "bytes" "encoding/json" + "fmt" + "text/template" "time" "github.com/thomaspoignant/go-feature-flag/ffcontext" @@ -83,15 +86,50 @@ type FeatureEvent struct { Metadata FeatureEventMetadata `json:"metadata,omitempty" parquet:"name=metadata, type=MAP, keytype=BYTE_ARRAY, keyconvertedtype=UTF8, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` } -// MarshalInterface marshals all interface type fields in FeatureEvent into JSON-encoded string. -func (f *FeatureEvent) MarshalInterface() error { - if f == nil { - return nil +// GetKey returns the key of the event +func (f FeatureEvent) GetKey() string { + return f.Key +} + +// GetUserKey returns the user key of the event +func (f FeatureEvent) GetUserKey() string { + return f.UserKey +} + +// GetCreationDate returns the creationDate of the event. +func (f FeatureEvent) GetCreationDate() int64 { + return f.CreationDate +} + +func (f FeatureEvent) FormatInCSV(csvTemplate *template.Template) ([]byte, error) { + var buf bytes.Buffer + err := csvTemplate.Execute(&buf, struct { + FeatureEvent + FormattedDate string + }{ + FeatureEvent: f, + FormattedDate: time.Unix(f.GetCreationDate(), 0).Format(time.RFC3339), + }) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (f FeatureEvent) FormatInJSON() ([]byte, error) { + b, err := json.Marshal(f) + b = append(b, []byte("\n")...) + return b, err +} + +// ConvertValueForParquet converts the value of the event to a string to be stored in a parquet file. +func (f FeatureEvent) ConvertValueForParquet() (string, error) { + if f.Value == nil { + return "", fmt.Errorf("no value to convert, returning empty string") } b, err := json.Marshal(f.Value) if err != nil { - return err + return "", err } - f.Value = string(b) - return nil + return string(b), nil } diff --git a/exporter/feature_event_test.go b/exporter/feature_event_test.go index d84c2e9bb13..350a2a76a5a 100644 --- a/exporter/feature_event_test.go +++ b/exporter/feature_event_test.go @@ -1,7 +1,7 @@ package exporter_test import ( - "encoding/json" + "fmt" "testing" "time" @@ -123,24 +123,35 @@ func TestFeatureEvent_MarshalInterface(t *testing.T) { wantErr: true, }, { - name: "nil featureEvent", - featureEvent: nil, + name: "nil value", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: nil, + Default: false, + }, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := tt.featureEvent.MarshalInterface(); (err != nil) != tt.wantErr { + val, err := tt.featureEvent.ConvertValueForParquet() + if (err != nil) != tt.wantErr { t.Errorf("FeatureEvent.MarshalInterface() error = %v, wantErr %v", err, tt.wantErr) return } if tt.want != nil { - assert.Equal(t, tt.want, tt.featureEvent) + assert.Equal(t, tt.want.Value, val) } }) } } -func TestFeatureEvent_MarshalJSON(t *testing.T) { +func TestFeatureEvent_FormatInJSON(t *testing.T) { tests := []struct { name string featureEvent *exporter.FeatureEvent @@ -165,7 +176,7 @@ func TestFeatureEvent_MarshalJSON(t *testing.T) { Default: false, Metadata: map[string]interface{}{}, }, - want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false}`, + want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"bool":true,"float":1.23,"int":1,"string":"string"},"default":false,"version":"","source":""}`, wantErr: assert.NoError, }, { @@ -185,7 +196,7 @@ func TestFeatureEvent_MarshalJSON(t *testing.T) { }, Default: false, }, - want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false}`, + want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"bool":true,"float":1.23,"int":1,"string":"string"},"default":false,"version":"","source":""}`, wantErr: assert.NoError, }, { @@ -210,17 +221,98 @@ func TestFeatureEvent_MarshalJSON(t *testing.T) { "metadata3": true, }, }, - want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false,"metadata":{"metadata1":"metadata1","metadata2":24,"metadata3":true}}`, + want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"bool":true,"float":1.23,"int":1,"string":"string"},"default":false,"version":"","source":"","metadata":{"metadata1":"metadata1","metadata2":24,"metadata3":true}}`, wantErr: assert.NoError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := json.Marshal(tt.featureEvent) + got, err := tt.featureEvent.FormatInJSON() tt.wantErr(t, err) - if err != nil { + if err == nil { + fmt.Println(string(got)) assert.JSONEq(t, tt.want, string(got)) } }) } } + +func TestFeatureEvent_GetKey(t *testing.T) { + tests := []struct { + name string + featureEvent *exporter.FeatureEvent + want string + }{ + { + name: "return existing key", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + }, + want: "random-key", + }, + { + name: "empty key", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "", + Variation: "Default", + Value: nil, + Default: false, + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.featureEvent.GetKey()) + }) + } +} + +func TestFeatureEvent_GetUserKey(t *testing.T) { + tests := []struct { + name string + featureEvent *exporter.FeatureEvent `` + want string + }{ + { + name: "return existing key", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + }, + want: "ABCD", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.featureEvent.GetUserKey()) + }) + } +} diff --git a/exporter/fileexporter/exporter.go b/exporter/fileexporter/exporter.go index b9bb212bd4e..cf7f59e98f9 100644 --- a/exporter/fileexporter/exporter.go +++ b/exporter/fileexporter/exporter.go @@ -56,7 +56,7 @@ type Exporter struct { func (f *Exporter) Export( _ context.Context, _ *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { // Parse the template only once f.initTemplates.Do(func() { @@ -91,7 +91,7 @@ func (f *Exporter) Export( filePath = filename } else { // Ensure OutputDir exists or create it - // nolint: gosec + // nolint:gosec if err := os.MkdirAll(outputDir, 0755); err != nil { return fmt.Errorf("failed to create output directory: %v", err) } @@ -99,9 +99,9 @@ func (f *Exporter) Export( } if f.Format == "parquet" { - return f.writeParquet(filePath, featureEvents) + return f.writeParquet(filePath, events) } - return f.writeFile(filePath, featureEvents) + return f.writeFile(filePath, events) } // IsBulk return false if we should directly send the data as soon as it is produce @@ -110,24 +110,25 @@ func (f *Exporter) IsBulk() bool { return true } -func (f *Exporter) writeFile(filePath string, featureEvents []exporter.FeatureEvent) error { - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) // nolint: gosec +func (f *Exporter) writeFile(filePath string, events []exporter.ExportableEvent) error { + //nolint:gosec + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return err } defer func() { _ = file.Close() }() - for _, event := range featureEvents { + for _, event := range events { var line []byte var err error // Convert the line in the right format switch f.Format { case "csv": - line, err = exporter.FormatEventInCSV(f.csvTemplate, event) + line, err = event.FormatInCSV(f.csvTemplate) case "json": - line, err = exporter.FormatEventInJSON(event) + line, err = event.FormatInJSON() default: - line, err = exporter.FormatEventInJSON(event) + line, err = event.FormatInJSON() } // Handle error and write line into the file @@ -142,7 +143,27 @@ func (f *Exporter) writeFile(filePath string, featureEvents []exporter.FeatureEv return nil } -func (f *Exporter) writeParquet(filePath string, featureEvents []exporter.FeatureEvent) error { +func (f *Exporter) writeParquet(filePath string, events []exporter.ExportableEvent) error { + parquetFeatureEvents := make([]exporter.FeatureEvent, 0) + parquetTrackingEvents := make([]exporter.TrackingEvent, 0) + for _, event := range events { + switch ev := any(event).(type) { + case exporter.FeatureEvent: + parquetFeatureEvents = append(parquetFeatureEvents, ev) + case exporter.TrackingEvent: + parquetTrackingEvents = append(parquetTrackingEvents, ev) + default: + // do nothing + } + } + if len(parquetTrackingEvents) > 0 { + return f.writeParquetTrackingEvent(filePath, parquetTrackingEvents) + } + return f.writeParquetFeatureEvent(filePath, parquetFeatureEvents) +} + +// writeParquetFeatureEvent writes the feature events in a parquet file +func (f *Exporter) writeParquetFeatureEvent(filePath string, events []exporter.FeatureEvent) error { fw, err := local.NewLocalFileWriter(filePath) if err != nil { return err @@ -159,12 +180,44 @@ func (f *Exporter) writeParquet(filePath string, featureEvents []exporter.Featur pw.CompressionType = ct } - for _, event := range featureEvents { - if err := event.MarshalInterface(); err != nil { + for _, event := range events { + eventValue, err := event.ConvertValueForParquet() + if err != nil { return err } + event.Value = eventValue if err = pw.Write(event); err != nil { - return fmt.Errorf("error while writing the export file: %v", err) + return fmt.Errorf("error while writing the parquet export file: %v", err) + } + } + + return pw.WriteStop() +} + +// writeParquetTrackingEvent writes the tracking events in a parquet file +func (f *Exporter) writeParquetTrackingEvent( + filePath string, + events []exporter.TrackingEvent, +) error { + fw, err := local.NewLocalFileWriter(filePath) + if err != nil { + return err + } + defer func() { _ = fw.Close() }() + + pw, err := writer.NewParquetWriter(fw, new(exporter.TrackingEvent), int64(runtime.NumCPU())) + if err != nil { + return err + } + + pw.CompressionType = parquet.CompressionCodec_SNAPPY + if ct, err := parquet.CompressionCodecFromString(f.ParquetCompressionCodec); err == nil { + pw.CompressionType = ct + } + + for _, event := range events { + if err = pw.Write(event); err != nil { + return fmt.Errorf("error while writing the parquet export file: %v", err) } } diff --git a/exporter/fileexporter/exporter_test.go b/exporter/fileexporter/exporter_test.go index 2cc91aae66b..af87c6ce399 100644 --- a/exporter/fileexporter/exporter_test.go +++ b/exporter/fileexporter/exporter_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/thomaspoignant/go-feature-flag/exporter" "github.com/thomaspoignant/go-feature-flag/exporter/fileexporter" + "github.com/thomaspoignant/go-feature-flag/ffcontext" "github.com/thomaspoignant/go-feature-flag/utils/fflog" "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/parquet" @@ -33,15 +34,17 @@ func TestFile_Export(t *testing.T) { CsvTemplate string OutputDir string ParquetCompressionCodec string + EventType string } type args struct { - logger *fflog.FFLogger - featureEvents []exporter.FeatureEvent + logger *fflog.FFLogger + events []exporter.ExportableEvent } type expected struct { - fileNameRegex string - content string - featureEvents []exporter.FeatureEvent + fileNameRegex string + content string + featureEvents []exporter.FeatureEvent + trackingEvents []exporter.TrackingEvent } tests := []struct { name string @@ -57,12 +60,12 @@ func TestFile_Export(t *testing.T) { wantErr: false, fields: fields{}, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -80,12 +83,12 @@ func TestFile_Export(t *testing.T) { Format: "csv", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -104,12 +107,12 @@ func TestFile_Export(t *testing.T) { ParquetCompressionCodec: parquet.CompressionCodec_SNAPPY.String(), }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", Metadata: map[string]interface{}{"test": "test"}, }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -129,6 +132,44 @@ func TestFile_Export(t *testing.T) { }, }, }, + { + name: "all default parquet tracking events", + wantErr: false, + fields: fields{ + Format: "parquet", + ParquetCompressionCodec: parquet.CompressionCodec_SNAPPY.String(), + Filename: "tracking-{{ .Hostname}}-{{ .Timestamp}}.parquet", + EventType: "tracking", + }, + args: args{ + + events: []exporter.ExportableEvent{ + exporter.TrackingEvent{ + Kind: "feature", + ContextKind: "anonymous", + UserKey: "xxx", + CreationDate: 1617970547, + Key: "what-ever-you-want", + EvaluationContext: ffcontext.NewEvaluationContext("xxx-xxx-xxx").ToMap(), + TrackingDetails: map[string]interface{}{"foo": "bar"}, + }, + }, + }, + expected: expected{ + fileNameRegex: "^tracking-" + hostname + "-[0-9]*\\.parquet$", + trackingEvents: []exporter.TrackingEvent{ + { + Kind: "feature", + ContextKind: "anonymous", + UserKey: "xxx", + CreationDate: 1617970547, + Key: "what-ever-you-want", + EvaluationContext: ffcontext.NewEvaluationContext("xxx-xxx-xxx").ToMap(), + TrackingDetails: map[string]interface{}{"foo": "bar"}, + }, + }, + }, + }, { name: "custom CSV format", wantErr: false, @@ -137,12 +178,12 @@ func TestFile_Export(t *testing.T) { CsvTemplate: "{{ .Kind}};{{ .ContextKind}}\n", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -161,8 +202,8 @@ func TestFile_Export(t *testing.T) { ParquetCompressionCodec: parquet.CompressionCodec_UNCOMPRESSED.String(), }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", @@ -207,12 +248,12 @@ func TestFile_Export(t *testing.T) { Filename: "{{ .Format}}-test-{{ .Timestamp}}", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -230,12 +271,12 @@ func TestFile_Export(t *testing.T) { Format: "xxx", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -253,12 +294,12 @@ func TestFile_Export(t *testing.T) { OutputDir: filepath.Join(tempDir, "non-existent-dir"), }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -276,12 +317,12 @@ func TestFile_Export(t *testing.T) { Filename: "{{ .InvalidField}}", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -296,12 +337,12 @@ func TestFile_Export(t *testing.T) { CsvTemplate: "{{ .Foo}}", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -316,8 +357,8 @@ func TestFile_Export(t *testing.T) { OutputDir: filepath.Join(tempDir, "invalid-permissions-dir"), }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -342,8 +383,8 @@ func TestFile_Export(t *testing.T) { OutputDir: filepath.Join(tempDir, "invalid-parent-dir"), }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -368,12 +409,12 @@ func TestFile_Export(t *testing.T) { OutputDir: filepath.Join(tempDir, "dir-with-trailing-slash") + "/", }, args: args{ - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -408,7 +449,7 @@ func TestFile_Export(t *testing.T) { CsvTemplate: tt.fields.CsvTemplate, ParquetCompressionCodec: tt.fields.ParquetCompressionCodec, } - err := f.Export(context.Background(), tt.args.logger, tt.args.featureEvents) + err := f.Export(context.Background(), tt.args.logger, tt.args.events) if tt.wantErr { assert.Error(t, err, "export method should error") return @@ -425,21 +466,40 @@ func TestFile_Export(t *testing.T) { assert.Regexp(t, tt.expected.fileNameRegex, files[0].Name(), "Invalid file name") if tt.fields.Format == "parquet" { - fr, err := local.NewLocalFileReader(outputDir + "/" + files[0].Name()) - assert.NoError(t, err) - defer fr.Close() - pr, err := reader.NewParquetReader( - fr, - new(exporter.FeatureEvent), - int64(runtime.NumCPU()), - ) - assert.NoError(t, err) - defer pr.ReadStop() - gotFeatureEvents := make([]exporter.FeatureEvent, pr.GetNumRows()) - err = pr.Read(&gotFeatureEvents) - assert.NoError(t, err) - assert.ElementsMatch(t, tt.expected.featureEvents, gotFeatureEvents) - return + switch tt.fields.EventType { + case "tracking": + fr, err := local.NewLocalFileReader(outputDir + "/" + files[0].Name()) + assert.NoError(t, err) + defer fr.Close() + pr, err := reader.NewParquetReader( + fr, + new(exporter.TrackingEvent), + int64(runtime.NumCPU()), + ) + assert.NoError(t, err) + defer pr.ReadStop() + gotFeatureEvents := make([]exporter.TrackingEvent, pr.GetNumRows()) + err = pr.Read(&gotFeatureEvents) + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expected.trackingEvents, gotFeatureEvents) + return + default: + fr, err := local.NewLocalFileReader(outputDir + "/" + files[0].Name()) + assert.NoError(t, err) + defer fr.Close() + pr, err := reader.NewParquetReader( + fr, + new(exporter.FeatureEvent), + int64(runtime.NumCPU()), + ) + assert.NoError(t, err) + defer pr.ReadStop() + gotFeatureEvents := make([]exporter.FeatureEvent, pr.GetNumRows()) + err = pr.Read(&gotFeatureEvents) + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expected.featureEvents, gotFeatureEvents) + return + } } expectedContent, _ := os.ReadFile(tt.expected.content) @@ -455,15 +515,16 @@ func TestFile_Export(t *testing.T) { } func TestFile_IsBulk(t *testing.T) { - exporter := fileexporter.Exporter{} - assert.True(t, exporter.IsBulk(), "DeprecatedExporter is a bulk exporter") + e := fileexporter.Exporter{} + assert.True(t, e.IsBulk(), "DeprecatedExporterV1 is a bulk exporter") } func TestExportWithoutOutputDir(t *testing.T) { - featureEvents := []exporter.FeatureEvent{{ - Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", - Variation: "Default", Value: "YO", Default: false, Source: "SERVER", - }} + featureEvents := []exporter.ExportableEvent{ + exporter.FeatureEvent{ + Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", + Variation: "Default", Value: "YO", Default: false, Source: "SERVER", + }} filePrefix := "test-flag-variation-EXAMPLE-" e := fileexporter.Exporter{ diff --git a/exporter/gcstorageexporter/exporter.go b/exporter/gcstorageexporter/exporter.go index 99cd65faf67..fb4430a8283 100644 --- a/exporter/gcstorageexporter/exporter.go +++ b/exporter/gcstorageexporter/exporter.go @@ -57,7 +57,7 @@ func (f *Exporter) IsBulk() bool { func (f *Exporter) Export( ctx context.Context, logger *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { // Init google storage client client, err := storage.NewClient(ctx, f.Options...) @@ -85,7 +85,7 @@ func (f *Exporter) Export( CsvTemplate: f.CsvTemplate, ParquetCompressionCodec: f.ParquetCompressionCodec, } - err = fileExporter.Export(ctx, logger, featureEvents) + err = fileExporter.Export(ctx, logger, events) if err != nil { return err } diff --git a/exporter/gcstorageexporter/exporter_test.go b/exporter/gcstorageexporter/exporter_test.go index 118b1455871..1553f339708 100644 --- a/exporter/gcstorageexporter/exporter_test.go +++ b/exporter/gcstorageexporter/exporter_test.go @@ -31,7 +31,7 @@ func TestGoogleStorage_Export(t *testing.T) { tests := []struct { name string fields fields - events []exporter.FeatureEvent + events []exporter.ExportableEvent wantErr bool expectedName string }{ @@ -40,8 +40,8 @@ func TestGoogleStorage_Export(t *testing.T) { fields: fields{ Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -54,8 +54,8 @@ func TestGoogleStorage_Export(t *testing.T) { Path: "random/path", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -68,8 +68,8 @@ func TestGoogleStorage_Export(t *testing.T) { Format: "csv", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -83,8 +83,8 @@ func TestGoogleStorage_Export(t *testing.T) { CsvTemplate: "{{ .Kind}};{{ .ContextKind}}\n", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -98,8 +98,8 @@ func TestGoogleStorage_Export(t *testing.T) { Filename: "{{ .Format}}-test-{{ .Timestamp}}", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -112,8 +112,8 @@ func TestGoogleStorage_Export(t *testing.T) { Format: "xxx", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -125,8 +125,8 @@ func TestGoogleStorage_Export(t *testing.T) { fields: fields{ Format: "xxx", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -139,8 +139,8 @@ func TestGoogleStorage_Export(t *testing.T) { Filename: "{{ .InvalidField}}", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -153,8 +153,8 @@ func TestGoogleStorage_Export(t *testing.T) { Format: "csv", CsvTemplate: "{{ .Foo}}", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -176,7 +176,7 @@ func TestGoogleStorage_Export(t *testing.T) { }, } - // init DeprecatedExporter + // init DeprecatedExporterV1 f := gcstorageexporter.Exporter{ Bucket: tt.fields.Bucket, Options: []option.ClientOption{ diff --git a/exporter/kafkaexporter/exporter.go b/exporter/kafkaexporter/exporter.go index 97627a62586..656a44287e5 100644 --- a/exporter/kafkaexporter/exporter.go +++ b/exporter/kafkaexporter/exporter.go @@ -48,7 +48,7 @@ type Exporter struct { func (e *Exporter) Export( _ context.Context, _ *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { if e.sender == nil { err := e.initializeProducer() @@ -57,8 +57,8 @@ func (e *Exporter) Export( } } - messages := make([]*sarama.ProducerMessage, 0, len(featureEvents)) - for _, event := range featureEvents { + messages := make([]*sarama.ProducerMessage, 0, len(events)) + for _, event := range events { data, err := e.formatMessage(event) if err != nil { return fmt.Errorf("format: %w", err) @@ -66,7 +66,7 @@ func (e *Exporter) Export( messages = append(messages, &sarama.ProducerMessage{ Topic: e.Settings.Topic, - Key: sarama.StringEncoder(event.UserKey), + Key: sarama.StringEncoder(event.GetUserKey()), Value: sarama.ByteEncoder(data), }) } @@ -113,7 +113,7 @@ func (e *Exporter) initializeProducer() error { } // formatMessage returns the event encoded in the selected format. Will always use JSON for now. -func (e *Exporter) formatMessage(event exporter.FeatureEvent) ([]byte, error) { +func (e *Exporter) formatMessage(event exporter.ExportableEvent) ([]byte, error) { switch e.Format { case formatJSON: fallthrough diff --git a/exporter/kafkaexporter/exporter_test.go b/exporter/kafkaexporter/exporter_test.go index ef361b0d32d..a99c3a7721e 100644 --- a/exporter/kafkaexporter/exporter_test.go +++ b/exporter/kafkaexporter/exporter_test.go @@ -25,19 +25,19 @@ func (s *messageSenderMock) SendMessages(msgs []*sarama.ProducerMessage) error { func TestExporter_IsBulk(t *testing.T) { exp := Exporter{} - assert.False(t, exp.IsBulk(), "DeprecatedExporter is not a bulk exporter") + assert.False(t, exp.IsBulk(), "DeprecatedExporterV1 is not a bulk exporter") } func TestExporter_Export(t *testing.T) { const mockTopic = "mockTopic" tests := []struct { - name string - format string - dialer func(addrs []string, config *sarama.Config) (MessageSender, error) - featureEvents []exporter.FeatureEvent - wantErr bool - settings Settings + name string + format string + dialer func(addrs []string, config *sarama.Config) (MessageSender, error) + events []exporter.ExportableEvent + wantErr bool + settings Settings }{ { name: "should receive an error if dial failed", @@ -65,12 +65,12 @@ func TestExporter_Export(t *testing.T) { name: "should receive an event with a valid feature event", format: "json", wantErr: false, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCDEF", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -87,12 +87,12 @@ func TestExporter_Export(t *testing.T) { name: "should default to JSON format if none provided", format: "", // Should default to JSON and generate a valid message wantErr: false, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCDEF", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -109,12 +109,12 @@ func TestExporter_Export(t *testing.T) { name: "should return an error if the publisher is returning an error", format: "json", wantErr: true, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCDEF", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -144,7 +144,7 @@ func TestExporter_Export(t *testing.T) { } logger := &fflog.FFLogger{LeveledLogger: slog.Default()} - err := exp.Export(context.Background(), logger, tt.featureEvents) + err := exp.Export(context.Background(), logger, tt.events) if tt.wantErr { assert.Error(t, err) return @@ -152,12 +152,12 @@ func TestExporter_Export(t *testing.T) { assert.NoError(t, err) - want := make([]*sarama.ProducerMessage, len(tt.featureEvents)) - for index, event := range tt.featureEvents { + want := make([]*sarama.ProducerMessage, len(tt.events)) + for index, event := range tt.events { messageBody, _ := json.Marshal(event) want[index] = &sarama.ProducerMessage{ Topic: mockTopic, - Key: sarama.StringEncoder(event.UserKey), + Key: sarama.StringEncoder(event.GetUserKey()), Value: sarama.ByteEncoder(messageBody), } } diff --git a/exporter/kinesisexporter/exporter.go b/exporter/kinesisexporter/exporter.go index 64729b87475..7ffd8f3a280 100644 --- a/exporter/kinesisexporter/exporter.go +++ b/exporter/kinesisexporter/exporter.go @@ -19,7 +19,7 @@ const ( Mb = 1024 * 1024 ) -var DefaultPartitionKey = func(context context.Context, _ exporter.FeatureEvent) string { +var DefaultPartitionKey = func(context context.Context, _ exporter.ExportableEvent) string { context.Value("feature") return "default" @@ -67,7 +67,7 @@ type Exporter struct { sender MessageSender } -type PartitionKeyFunc = func(context.Context, exporter.FeatureEvent) string +type PartitionKeyFunc = func(context.Context, exporter.ExportableEvent) string type Settings struct { StreamName *string @@ -151,7 +151,7 @@ func (e *Exporter) initializeProducer(ctx context.Context) error { func (e *Exporter) Export( ctx context.Context, logger *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + featureEvents []exporter.ExportableEvent, ) error { err := e.initializeProducer(ctx) if err != nil { @@ -223,7 +223,7 @@ func (e *Exporter) Export( } // formatMessage returns the event encoded in the selected format. Will always use JSON for now. -func (e *Exporter) formatMessage(event exporter.FeatureEvent) ([]byte, error) { +func (e *Exporter) formatMessage(event exporter.ExportableEvent) ([]byte, error) { switch e.Format { case formatJSON: fallthrough diff --git a/exporter/kinesisexporter/exporter_test.go b/exporter/kinesisexporter/exporter_test.go index eb041534f79..fe7ce6ac8fc 100644 --- a/exporter/kinesisexporter/exporter_test.go +++ b/exporter/kinesisexporter/exporter_test.go @@ -17,7 +17,7 @@ import ( func TestExporter_IsBulk(t *testing.T) { exp := Exporter{} - assert.False(t, exp.IsBulk(), "DeprecatedExporter is not a bulk exporter") + assert.False(t, exp.IsBulk(), "DeprecatedExporterV1 is not a bulk exporter") } func TestExporter_ExportBasicWithStreamName(t *testing.T) { @@ -34,10 +34,10 @@ func TestExporter_ExportBasicWithStreamName(t *testing.T) { err := exp.Export( context.Background(), logger, - []exporter.FeatureEvent{ - *NewFeatureEvent(), - *NewFeatureEvent(), - *NewFeatureEvent(), + []exporter.ExportableEvent{ + NewFeatureEvent(), + NewFeatureEvent(), + NewFeatureEvent(), }, ) @@ -67,10 +67,10 @@ func TestExporter_ExportBasicWithStreamArn(t *testing.T) { err := exp.Export( context.Background(), logger, - []exporter.FeatureEvent{ - *NewFeatureEvent(), - *NewFeatureEvent(), - *NewFeatureEvent(), + []exporter.ExportableEvent{ + NewFeatureEvent(), + NewFeatureEvent(), + NewFeatureEvent(), }, ) @@ -101,7 +101,7 @@ func TestExporter_ShouldRaiseErrorIfNoStreamIsSpecified(t *testing.T) { err := exp.Export( context.Background(), logger, - []exporter.FeatureEvent{*NewFeatureEvent()}, + []exporter.ExportableEvent{NewFeatureEvent()}, ) assert.Error(t, err) @@ -115,7 +115,7 @@ func TestExporter_ExportAWSConfigurationCustomisation(t *testing.T) { sender: &mock, Settings: NewSettings( WithStreamName("test-stream"), - WithPartitionKey(func(context.Context, exporter.FeatureEvent) string { + WithPartitionKey(func(context.Context, exporter.ExportableEvent) string { return "test-key" }), ), @@ -129,8 +129,8 @@ func TestExporter_ExportAWSConfigurationCustomisation(t *testing.T) { err := exp.Export( context.Background(), logger, - []exporter.FeatureEvent{ - *NewFeatureEvent(), + []exporter.ExportableEvent{ + NewFeatureEvent(), }, ) @@ -153,8 +153,8 @@ func TestExporter_ExportSenderError(t *testing.T) { err := exp.Export( context.Background(), logger, - []exporter.FeatureEvent{ - *NewFeatureEvent(), + []exporter.ExportableEvent{ + NewFeatureEvent(), }, ) @@ -164,28 +164,28 @@ func TestExporter_ExportSenderError(t *testing.T) { func TestExporterSettingsCreation(t *testing.T) { { settings := NewSettings() - assert.Equal(t, settings.PartitionKey(context.TODO(), *NewFeatureEvent()), "default") + assert.Equal(t, settings.PartitionKey(context.TODO(), NewFeatureEvent()), "default") assert.Nil(t, settings.StreamName) assert.Nil(t, settings.StreamArn) assert.Nil(t, settings.ExplicitHashKey) } { settings := NewSettings(WithStreamArn("test-stream-arn")) - assert.Equal(t, settings.PartitionKey(context.TODO(), *NewFeatureEvent()), "default") + assert.Equal(t, settings.PartitionKey(context.TODO(), NewFeatureEvent()), "default") assert.Nil(t, settings.StreamName) assert.Equal(t, *settings.StreamArn, "test-stream-arn") assert.Nil(t, settings.ExplicitHashKey) } { settings := NewSettings(WithStreamName("test-stream-name")) - assert.Equal(t, settings.PartitionKey(context.TODO(), *NewFeatureEvent()), "default") + assert.Equal(t, settings.PartitionKey(context.TODO(), NewFeatureEvent()), "default") assert.Equal(t, *settings.StreamName, "test-stream-name") assert.Nil(t, settings.StreamArn) assert.Nil(t, settings.ExplicitHashKey) } { settings := NewSettings(WithExplicitHashKey("test-explicit-hash-key")) - assert.Equal(t, settings.PartitionKey(context.TODO(), *NewFeatureEvent()), "default") + assert.Equal(t, settings.PartitionKey(context.TODO(), NewFeatureEvent()), "default") assert.Nil(t, settings.StreamName) assert.Nil(t, settings.StreamArn) assert.Equal(t, *settings.ExplicitHashKey, "test-explicit-hash-key") @@ -196,10 +196,10 @@ func TestExporterSettingsCreation(t *testing.T) { WithStreamArn("test-stream-arn"), WithExplicitHashKey("test-explicit-hash-key"), WithPartitionKey( - func(_ context.Context, _ exporter.FeatureEvent) string { return "non-default" }, + func(_ context.Context, _ exporter.ExportableEvent) string { return "non-default" }, ), ) - assert.Equal(t, settings.PartitionKey(context.TODO(), *NewFeatureEvent()), "non-default") + assert.Equal(t, settings.PartitionKey(context.TODO(), NewFeatureEvent()), "non-default") assert.Nil(t, settings.StreamName) // overwritten by streamArn assert.Equal(t, *settings.StreamArn, "test-stream-arn") assert.Equal(t, *settings.ExplicitHashKey, "test-explicit-hash-key") @@ -215,7 +215,16 @@ func TestExporterSettingsCreation(t *testing.T) { } func TestHugeMessageExportFlow(t *testing.T) { - event := NewFeatureEvent() + event := exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: "YO", + Default: false, + } event.Value = string(make([]byte, Mb)) mock := MockKinesisSender{} @@ -231,11 +240,11 @@ func TestHugeMessageExportFlow(t *testing.T) { err := exp.Export( context.Background(), logger, - []exporter.FeatureEvent{ - *event, - *event, - *event, - *event, + []exporter.ExportableEvent{ + event, + event, + event, + event, }, ) @@ -243,7 +252,7 @@ func TestHugeMessageExportFlow(t *testing.T) { assert.Len(t, mock.PutRecordsInputs, 0) } -func NewFeatureEvent() *exporter.FeatureEvent { +func NewFeatureEvent() exporter.ExportableEvent { return &exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", diff --git a/exporter/logsexporter/exporter.go b/exporter/logsexporter/exporter.go index 91d8c199961..9120b1fac6f 100644 --- a/exporter/logsexporter/exporter.go +++ b/exporter/logsexporter/exporter.go @@ -1,11 +1,9 @@ package logsexporter import ( - "bytes" "context" "sync" "text/template" - "time" "github.com/thomaspoignant/go-feature-flag/exporter" "github.com/thomaspoignant/go-feature-flag/utils/fflog" @@ -34,28 +32,22 @@ type Exporter struct { func (f *Exporter) Export( _ context.Context, logger *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { f.initTemplates.Do(func() { // Remove below after deprecation of Format if f.LogFormat == "" && f.Format != "" { f.LogFormat = f.Format } - f.logTemplate = exporter.ParseTemplate("logFormat", f.LogFormat, defaultLoggerFormat) }) - for _, event := range featureEvents { - var log bytes.Buffer - err := f.logTemplate.Execute(&log, struct { - exporter.FeatureEvent - FormattedDate string - }{FeatureEvent: event, FormattedDate: time.Unix(event.CreationDate, 0).Format(time.RFC3339)}) - - logger.Info(log.String()) + for _, event := range events { + log, err := event.FormatInCSV(f.logTemplate) if err != nil { return err } + logger.Info(string(log)) } return nil } diff --git a/exporter/logsexporter/exporter_test.go b/exporter/logsexporter/exporter_test.go index 593158151ee..86f76bc0d03 100644 --- a/exporter/logsexporter/exporter_test.go +++ b/exporter/logsexporter/exporter_test.go @@ -19,7 +19,7 @@ func TestLog_Export(t *testing.T) { LogFormat string } type args struct { - featureEvents []exporter.FeatureEvent + featureEvents []exporter.ExportableEvent } tests := []struct { name string @@ -31,8 +31,8 @@ func TestLog_Export(t *testing.T) { { name: "Default format", fields: fields{LogFormat: ""}, - args: args{featureEvents: []exporter.FeatureEvent{ - { + args: args{featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -44,8 +44,8 @@ func TestLog_Export(t *testing.T) { fields: fields{ LogFormat: "key=\"{{ .Key}}\"", }, - args: args{featureEvents: []exporter.FeatureEvent{ - { + args: args{featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -57,8 +57,8 @@ func TestLog_Export(t *testing.T) { fields: fields{ LogFormat: "key=\"{{ .Key}\" [{{ .FormattedDate}}]", }, - args: args{featureEvents: []exporter.FeatureEvent{ - { + args: args{featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -70,8 +70,8 @@ func TestLog_Export(t *testing.T) { fields: fields{ LogFormat: "key=\"{{ .UnknownKey}}\" [{{ .FormattedDate}}]", }, - args: args{featureEvents: []exporter.FeatureEvent{ - { + args: args{featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -96,7 +96,7 @@ func TestLog_Export(t *testing.T) { return } - assert.NoError(t, err, "DeprecatedExporter should not throw errors") + assert.NoError(t, err, "DeprecatedExporterV1 should not throw errors") logContent, _ := os.ReadFile(logFile.Name()) assert.Regexp(t, tt.expectedLog, string(logContent)) diff --git a/exporter/manager.go b/exporter/manager.go index 85c931dd6a5..dfdb912cd8a 100644 --- a/exporter/manager.go +++ b/exporter/manager.go @@ -10,19 +10,19 @@ import ( const DefaultExporterCleanQueueInterval = 1 * time.Minute -type Manager[T any] interface { +type Manager[T ExportableEvent] interface { AddEvent(event T) Start() Stop() } -type managerImpl[T any] struct { +type managerImpl[T ExportableEvent] struct { logger *fflog.FFLogger consumers []DataExporter[T] eventStore *EventStore[T] } -func NewManager[T any](ctx context.Context, exporters []Config, +func NewManager[T ExportableEvent](ctx context.Context, exporters []Config, exporterCleanQueueInterval time.Duration, logger *fflog.FFLogger) Manager[T] { if ctx == nil { ctx = context.Background() diff --git a/exporter/manager_test.go b/exporter/manager_test.go index 86a1a6a4244..49d02f7a7ab 100644 --- a/exporter/manager_test.go +++ b/exporter/manager_test.go @@ -32,6 +32,10 @@ func TestDataExporterManager_flushWithTime(t *testing.T) { name: "flushTime: deprecated exporter", mockExporter: &mock.ExporterDeprecated{Bulk: true}, }, + { + name: "flushTime: deprecated exporter v2", + mockExporter: &mock.ExporterDeprecatedV2{Bulk: true}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -67,12 +71,14 @@ func TestDataExporterManager_flushWithTime(t *testing.T) { ), } - for _, event := range inputEvents { + want := make([]exporter.ExportableEvent, len(inputEvents)) + for i, event := range inputEvents { dc.AddEvent(event) + want[i] = event } time.Sleep(500 * time.Millisecond) - assert.Equal(t, inputEvents, tt.mockExporter.GetExportedEvents()) + assert.Equal(t, want, tt.mockExporter.GetExportedEvents()) }) } } @@ -90,6 +96,10 @@ func TestDataExporterManager_flushWithNumberOfEvents(t *testing.T) { name: "flushWithNumberOfEvents: deprecated exporter", mockExporter: &mock.ExporterDeprecated{Bulk: true}, }, + { + name: "flushWithNumberOfEvents: deprecated exporter v2", + mockExporter: &mock.ExporterDeprecatedV2{Bulk: true}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -125,10 +135,12 @@ func TestDataExporterManager_flushWithNumberOfEvents(t *testing.T) { nil, )) } - for _, event := range inputEvents { + want := make([]exporter.ExportableEvent, len(inputEvents)) + for i, event := range inputEvents { dc.AddEvent(event) + want[i] = event } - assert.Equal(t, inputEvents[:100], tt.mockExporter.GetExportedEvents()) + assert.Equal(t, want[:100], tt.mockExporter.GetExportedEvents()) }) } } @@ -146,6 +158,10 @@ func TestDataExporterManager_defaultFlush(t *testing.T) { name: "deprecated exporter", mockExporter: &mock.ExporterDeprecated{Bulk: true}, }, + { + name: "deprecated exporter v2", + mockExporter: &mock.ExporterDeprecatedV2{Bulk: true}, + }, } for _, tt := range tests { @@ -178,10 +194,12 @@ func TestDataExporterManager_defaultFlush(t *testing.T) { nil, )) } - for _, event := range inputEvents { + want := make([]exporter.ExportableEvent, len(inputEvents)) + for i, event := range inputEvents { dc.AddEvent(event) + want[i] = event } - assert.Equal(t, inputEvents[:100000], tt.mockExporter.GetExportedEvents()) + assert.Equal(t, want[:100000], tt.mockExporter.GetExportedEvents()) }) } } @@ -212,11 +230,13 @@ func TestDataExporterManager_exporterReturnError(t *testing.T) { ffcontext.NewEvaluationContextBuilder("ABCD").AddCustom("anonymous", true).Build(), "random-key", "YO", "defaultVar", false, "", "SERVER", nil)) } - for _, event := range inputEvents { + want := make([]exporter.ExportableEvent, len(inputEvents)) + for i, event := range inputEvents { dc.AddEvent(event) + want[i] = event } // check that the first 100 events are exported - assert.Equal(t, inputEvents[:100], mockExporter.GetExportedEvents()[:100]) + assert.Equal(t, want[:100], mockExporter.GetExportedEvents()[:100]) handler.AssertMessage("error while exporting data: random err") } @@ -240,13 +260,15 @@ func TestDataExporterManager_nonBulkExporter(t *testing.T) { ffcontext.NewEvaluationContextBuilder("ABCD").AddCustom("anonymous", true).Build(), "random-key", "YO", "defaultVar", false, "", "SERVER", nil)) } - for _, event := range inputEvents { + want := make([]exporter.ExportableEvent, len(inputEvents)) + for i, event := range inputEvents { dc.AddEvent(event) + want[i] = event // we have to wait because we are opening a new thread to slow down the flag evaluation. time.Sleep(1 * time.Millisecond) } - assert.Equal(t, inputEvents[:100], mockExporter.GetExportedEvents()) + assert.Equal(t, want[:100], mockExporter.GetExportedEvents()) } func TestAddExporterMetadataFromContextToExporter(t *testing.T) { @@ -299,8 +321,14 @@ func TestAddExporterMetadataFromContextToExporter(t *testing.T) { time.Sleep(120 * time.Millisecond) assert.Equal(t, 1, len(mockExporter.GetExportedEvents())) - got := mockExporter.GetExportedEvents()[0].Metadata - assert.Equal(t, tt.want, got) + + switch val := mockExporter.GetExportedEvents()[0].(type) { + case exporter.FeatureEvent: + assert.Equal(t, tt.want, val.Metadata) + break + default: + assert.Fail(t, "The exported event is not a FeatureEvent") + } }) } } @@ -332,16 +360,18 @@ func TestDataExporterManager_multipleExporters(t *testing.T) { ffcontext.NewEvaluationContextBuilder("ABCD").AddCustom("anonymous", true).Build(), "random-key", "YO", "defaultVar", false, "", "SERVER", nil)) } - for _, event := range inputEvents { + want := make([]exporter.ExportableEvent, len(inputEvents)) + for i, event := range inputEvents { dc.AddEvent(event) + want[i] = event // we have to wait because we are opening a new thread to slow down the flag evaluation. time.Sleep(1 * time.Millisecond) } - assert.Equal(t, inputEvents[:100], mockExporter1.GetExportedEvents()) + assert.Equal(t, want[:100], mockExporter1.GetExportedEvents()) assert.Equal(t, 0, len(mockExporter2.GetExportedEvents())) time.Sleep(250 * time.Millisecond) - assert.Equal(t, inputEvents[:100], mockExporter2.GetExportedEvents()) + assert.Equal(t, want[:100], mockExporter2.GetExportedEvents()) } func TestDataExporterManager_multipleExportersWithDifferentFlushInterval(t *testing.T) { diff --git a/exporter/pubsubexporter/exporter.go b/exporter/pubsubexporter/exporter.go index 19523c8f483..d9fbd354495 100644 --- a/exporter/pubsubexporter/exporter.go +++ b/exporter/pubsubexporter/exporter.go @@ -39,7 +39,7 @@ type Exporter struct { func (e *Exporter) Export( ctx context.Context, _ *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { if e.publisher == nil { if err := e.initPublisher(ctx); err != nil { @@ -47,7 +47,7 @@ func (e *Exporter) Export( } } - for _, event := range featureEvents { + for _, event := range events { messageBody, err := json.Marshal(event) if err != nil { return err diff --git a/exporter/pubsubexporter/exporter_test.go b/exporter/pubsubexporter/exporter_test.go index b3c995a0b50..ce0b0e008cb 100644 --- a/exporter/pubsubexporter/exporter_test.go +++ b/exporter/pubsubexporter/exporter_test.go @@ -57,10 +57,10 @@ func TestExporter_Export(t *testing.T) { newClientFunc func(context.Context, string, ...option.ClientOption) (*pubsub.Client, error) } tests := []struct { - name string - fields fields - featureEvents []exporter.FeatureEvent - wantErr bool + name string + fields fields + events []exporter.ExportableEvent + wantErr bool }{ { name: "should publish a single message with the feature event", @@ -68,8 +68,8 @@ func TestExporter_Export(t *testing.T) { topic: topic, newClientFunc: defaultNewClientFunc, }, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -81,12 +81,12 @@ func TestExporter_Export(t *testing.T) { topic: topic, newClientFunc: defaultNewClientFunc, }, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature1", ContextKind: "anonymousUser1", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key1", Variation: "Default", Value: "YO", Default: false, }, - { + exporter.FeatureEvent{ Kind: "feature2", ContextKind: "anonymousUser2", UserKey: "ABCDEF", CreationDate: 1617970527, Key: "random-key2", Variation: "Default", Value: "YO", Default: true, }, @@ -106,8 +106,8 @@ func TestExporter_Export(t *testing.T) { return client, nil }, }, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -120,8 +120,8 @@ func TestExporter_Export(t *testing.T) { newClientFunc: defaultNewClientFunc, publishSettings: &pubsub.PublishSettings{CountThreshold: 123}, }, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -134,8 +134,8 @@ func TestExporter_Export(t *testing.T) { newClientFunc: defaultNewClientFunc, enableMessageOrdering: true, }, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -157,8 +157,8 @@ func TestExporter_Export(t *testing.T) { topic: "not-existing-topic", newClientFunc: defaultNewClientFunc, }, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -178,7 +178,7 @@ func TestExporter_Export(t *testing.T) { EnableMessageOrdering: tt.fields.enableMessageOrdering, newClientFunc: tt.fields.newClientFunc, } - err = e.Export(ctx, logger, tt.featureEvents) + err = e.Export(ctx, logger, tt.events) if tt.wantErr { assert.Error(t, err) @@ -186,7 +186,7 @@ func TestExporter_Export(t *testing.T) { } assert.NoError(t, err) - assertMessages(t, tt.featureEvents, server.Messages()) + assertMessages(t, tt.events, server.Messages()) assertPublisherSettings(t, tt.fields.publishSettings, e.publisher) assert.Equal(t, tt.fields.enableMessageOrdering, e.publisher.EnableMessageOrdering) }) @@ -203,7 +203,7 @@ func TestExporter_IsBulk(t *testing.T) { func assertMessages( t *testing.T, - expectedEvents []exporter.FeatureEvent, + expectedEvents []exporter.ExportableEvent, messages []*pstest.Message, ) { events := make([]exporter.FeatureEvent, len(messages)) diff --git a/exporter/s3exporter/exporter.go b/exporter/s3exporter/exporter.go index a6e2506bb3b..55931f0f6cd 100644 --- a/exporter/s3exporter/exporter.go +++ b/exporter/s3exporter/exporter.go @@ -15,7 +15,7 @@ import ( "github.com/thomaspoignant/go-feature-flag/utils/fflog" ) -// Deprecated: Please use s3exporterv2.DeprecatedExporter instead, it will use the go-aws-sdk-v2. +// Deprecated: Please use s3exporterv2.Exporter instead, it will use the go-aws-sdk-v2. type Exporter struct { // Bucket is the name of your Exporter Bucket. Bucket string @@ -59,7 +59,7 @@ type Exporter struct { func (f *Exporter) Export( ctx context.Context, logger *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { // init the s3 uploader if f.s3Uploader == nil { @@ -83,7 +83,7 @@ func (f *Exporter) Export( defer func() { _ = os.Remove(outputDir) }() // We call the File data exporter to get the file in the right format. - // Files will be put in the temp directory, so we will be able to upload them to DeprecatedExporter from there. + // Files will be put in the temp directory, so we will be able to upload them to Exporter from there. fileExporter := fileexporter.Exporter{ Format: f.Format, OutputDir: outputDir, @@ -91,12 +91,12 @@ func (f *Exporter) Export( CsvTemplate: f.CsvTemplate, ParquetCompressionCodec: f.ParquetCompressionCodec, } - err = fileExporter.Export(ctx, logger, featureEvents) + err = fileExporter.Export(ctx, logger, events) if err != nil { return err } - // Upload all the files in the folder to DeprecatedExporter + // Upload all the files in the folder to Export files, err := os.ReadDir(outputDir) if err != nil { return err diff --git a/exporter/s3exporter/exporter_test.go b/exporter/s3exporter/exporter_test.go index 219609102d1..d9322a19b51 100644 --- a/exporter/s3exporter/exporter_test.go +++ b/exporter/s3exporter/exporter_test.go @@ -27,7 +27,7 @@ func TestS3_Export(t *testing.T) { tests := []struct { name string fields fields - events []exporter.FeatureEvent + events []exporter.ExportableEvent wantErr bool expectedFile string expectedName string @@ -37,8 +37,8 @@ func TestS3_Export(t *testing.T) { fields: fields{ Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -47,13 +47,13 @@ func TestS3_Export(t *testing.T) { expectedName: "^/flag-variation-" + hostname + "-[0-9]*\\.json$", }, { - name: "With DeprecatedExporter Path", + name: "With DeprecatedExporterV1 Path", fields: fields{ S3Path: "random/path", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -67,8 +67,8 @@ func TestS3_Export(t *testing.T) { Format: "csv", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -83,8 +83,8 @@ func TestS3_Export(t *testing.T) { CsvTemplate: "{{ .Kind}};{{ .ContextKind}}\n", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -99,8 +99,8 @@ func TestS3_Export(t *testing.T) { Filename: "{{ .Format}}-test-{{ .Timestamp}}", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -114,8 +114,8 @@ func TestS3_Export(t *testing.T) { Format: "xxx", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -128,8 +128,8 @@ func TestS3_Export(t *testing.T) { fields: fields{ Format: "xxx", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -142,8 +142,8 @@ func TestS3_Export(t *testing.T) { Filename: "{{ .InvalidField}}", Bucket: "test", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -156,8 +156,8 @@ func TestS3_Export(t *testing.T) { Format: "csv", CsvTemplate: "{{ .Foo}}", }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -211,7 +211,7 @@ func Test_errSDK(t *testing.T) { err := f.Export( context.Background(), &fflog.FFLogger{LeveledLogger: slog.Default()}, - []exporter.FeatureEvent{}, + []exporter.ExportableEvent{}, ) assert.Error(t, err, "Empty AWS config should failed") } diff --git a/exporter/s3exporterv2/exporter.go b/exporter/s3exporterv2/exporter.go index 642a32b3e5f..c2e89e5ff03 100644 --- a/exporter/s3exporterv2/exporter.go +++ b/exporter/s3exporterv2/exporter.go @@ -85,7 +85,7 @@ func (f *Exporter) initializeUploader(ctx context.Context) error { func (f *Exporter) Export( ctx context.Context, logger *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { if ctx == nil { ctx = context.Background() @@ -105,7 +105,7 @@ func (f *Exporter) Export( defer func() { _ = os.Remove(outputDir) }() // We call the File data exporter to get the file in the right format. - // Files will be put in the temp directory, so we will be able to upload them to DeprecatedExporter from there. + // Files will be put in the temp directory, so we will be able to upload them to export from there. fileExporter := fileexporter.Exporter{ Format: f.Format, OutputDir: outputDir, @@ -113,12 +113,12 @@ func (f *Exporter) Export( CsvTemplate: f.CsvTemplate, ParquetCompressionCodec: f.ParquetCompressionCodec, } - err = fileExporter.Export(ctx, logger, featureEvents) + err = fileExporter.Export(ctx, logger, events) if err != nil { return err } - // Upload all the files in the folder to DeprecatedExporter + // Upload all the files in the folder to export files, err := os.ReadDir(outputDir) if err != nil { return err diff --git a/exporter/s3exporterv2/exporter_test.go b/exporter/s3exporterv2/exporter_test.go index 33a7a01be40..912fcd7b83c 100644 --- a/exporter/s3exporterv2/exporter_test.go +++ b/exporter/s3exporterv2/exporter_test.go @@ -30,7 +30,7 @@ func TestS3_Export(t *testing.T) { tests := []struct { name string fields fields - events []exporter.FeatureEvent + events []exporter.ExportableEvent wantErr bool expectedFile string expectedName string @@ -41,8 +41,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -56,8 +56,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: nil, }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -66,14 +66,14 @@ func TestS3_Export(t *testing.T) { expectedName: "^/flag-variation-" + hostname + "-[0-9]*\\.json$", }, { - name: "With DeprecatedExporter Path", + name: "With DeprecatedExporterV1 Path", fields: fields{ S3Path: "random/path", Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -88,8 +88,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -105,8 +105,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -122,8 +122,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -138,8 +138,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -153,8 +153,8 @@ func TestS3_Export(t *testing.T) { Format: "xxx", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -168,8 +168,8 @@ func TestS3_Export(t *testing.T) { Bucket: "test", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -183,8 +183,8 @@ func TestS3_Export(t *testing.T) { CsvTemplate: "{{ .Foo}}", Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -202,8 +202,8 @@ func TestS3_Export(t *testing.T) { }, Context: context.TODO(), }, - events: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, @@ -270,12 +270,12 @@ func Test_errSDK(t *testing.T) { err := f.Export( context.Background(), &fflog.FFLogger{LeveledLogger: slog.Default()}, - []exporter.FeatureEvent{}, + []exporter.ExportableEvent{}, ) assert.Error(t, err, "Empty AWS config should failed") } func TestS3_IsBulk(t *testing.T) { exporter := Exporter{} - assert.True(t, exporter.IsBulk(), "DeprecatedExporter is a bulk exporter") + assert.True(t, exporter.IsBulk(), "DeprecatedExporterV1 is a bulk exporter") } diff --git a/exporter/sqsexporter/exporter.go b/exporter/sqsexporter/exporter.go index d86676de638..9ba76d423a9 100644 --- a/exporter/sqsexporter/exporter.go +++ b/exporter/sqsexporter/exporter.go @@ -27,11 +27,11 @@ type Exporter struct { sqsService SQSSendMessageAPI } -// Export is sending SQS event for each featureEvents received. +// Export is sending SQS event for each events received. func (f *Exporter) Export( ctx context.Context, _ *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { if f.AwsConfig == nil { cfg, err := config.LoadDefaultConfig(ctx) @@ -51,7 +51,7 @@ func (f *Exporter) Export( }) } - for _, event := range featureEvents { + for _, event := range events { messageBody, err := json.Marshal(event) if err != nil { return err diff --git a/exporter/sqsexporter/exporter_test.go b/exporter/sqsexporter/exporter_test.go index 4f437342e0a..9b7aefd8961 100644 --- a/exporter/sqsexporter/exporter_test.go +++ b/exporter/sqsexporter/exporter_test.go @@ -32,7 +32,7 @@ func (s *SQSSendMessageAPIMock) SendMessage(ctx context.Context, func TestSQS_IsBulk(t *testing.T) { exporter := Exporter{} - assert.False(t, exporter.IsBulk(), "DeprecatedExporter is not a bulk exporter") + assert.False(t, exporter.IsBulk(), "DeprecatedExporterV1 is not a bulk exporter") } func TestExporter_Export(t *testing.T) { @@ -42,10 +42,10 @@ func TestExporter_Export(t *testing.T) { sqsService SQSSendMessageAPIMock } tests := []struct { - name string - fields fields - featureEvents []exporter.FeatureEvent - wantErr bool + name string + fields fields + events []exporter.ExportableEvent + wantErr bool }{ { name: "should return an error if no QueueURL provided", @@ -54,8 +54,8 @@ func TestExporter_Export(t *testing.T) { sqsService: SQSSendMessageAPIMock{}, }, wantErr: true, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -68,12 +68,12 @@ func TestExporter_Export(t *testing.T) { sqsService: SQSSendMessageAPIMock{}, }, wantErr: false, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCDEF", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -86,12 +86,12 @@ func TestExporter_Export(t *testing.T) { sqsService: SQSSendMessageAPIMock{}, }, wantErr: true, - featureEvents: []exporter.FeatureEvent{ - { + events: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCDEF", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, }, @@ -107,15 +107,15 @@ func TestExporter_Export(t *testing.T) { } logger := &fflog.FFLogger{LeveledLogger: slog.Default()} - err := f.Export(context.TODO(), logger, tt.featureEvents) + err := f.Export(context.TODO(), logger, tt.events) if tt.wantErr { assert.Error(t, err) return } assert.NoError(t, err) - want := make([]sqs.SendMessageInput, len(tt.featureEvents)) - for index, event := range tt.featureEvents { + want := make([]sqs.SendMessageInput, len(tt.events)) + for index, event := range tt.events { messageBody, _ := json.Marshal(event) want[index] = sqs.SendMessageInput{ MessageBody: aws.String(string(messageBody)), diff --git a/exporter/tracking_event.go b/exporter/tracking_event.go new file mode 100644 index 00000000000..261b6df9f17 --- /dev/null +++ b/exporter/tracking_event.go @@ -0,0 +1,73 @@ +package exporter + +import ( + "bytes" + "encoding/json" + "text/template" + "time" +) + +type TrackingEventDetails = map[string]interface{} + +// TrackingEvent represent an Event that we store in the data storage +// nolint:lll +type TrackingEvent struct { + // Kind for a feature event is feature. + // A feature event will only be generated if the trackEvents attribute of the flag is set to true. + Kind string `json:"kind" example:"feature" parquet:"name=kind, type=BYTE_ARRAY, convertedtype=UTF8"` + + // ContextKind is the kind of context which generated an event. This will only be "anonymousUser" for events generated + // on behalf of an anonymous user or the reserved word "user" for events generated on behalf of a non-anonymous user + ContextKind string `json:"contextKind,omitempty" example:"user" parquet:"name=contextKind, type=BYTE_ARRAY, convertedtype=UTF8"` + + // UserKey The key of the user object used in a feature flag evaluation. Details for the user object used in a feature + // flag evaluation as reported by the "feature" event are transmitted periodically with a separate index event. + UserKey string `json:"userKey" example:"94a25909-20d8-40cc-8500-fee99b569345" parquet:"name=userKey, type=BYTE_ARRAY, convertedtype=UTF8"` + + // CreationDate When the feature flag was requested at Unix epoch time in milliseconds. + CreationDate int64 `json:"creationDate" example:"1680246000011" parquet:"name=creationDate, type=INT64"` + + // Key of the feature flag requested. + Key string `json:"key" example:"my-feature-flag" parquet:"name=key, type=BYTE_ARRAY, convertedtype=UTF8"` + + // EvaluationContext contains the evaluation context used for the tracking + EvaluationContext map[string]any `json:"evaluationContext" parquet:"name=evaluationContext, type=MAP, keytype=BYTE_ARRAY, keyconvertedtype=UTF8, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` + + // TrackingDetails contains the details of the tracking event + TrackingDetails TrackingEventDetails `json:"trackingEventDetails" parquet:"name=trackingEventDetails, type=MAP, keytype=BYTE_ARRAY, keyconvertedtype=UTF8, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` +} + +func (f TrackingEvent) GetKey() string { + return f.Key +} + +// GetUserKey returns the user key of the event +func (f TrackingEvent) GetUserKey() string { + return f.UserKey +} + +// GetCreationDate returns the creationDate of the event. +func (f TrackingEvent) GetCreationDate() int64 { + return f.CreationDate +} + +func (f TrackingEvent) FormatInCSV(csvTemplate *template.Template) ([]byte, error) { + var buf bytes.Buffer + err := csvTemplate.Execute(&buf, struct { + TrackingEvent + FormattedDate string + }{ + TrackingEvent: f, + FormattedDate: time.Unix(f.GetCreationDate(), 0).Format(time.RFC3339), + }) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (f TrackingEvent) FormatInJSON() ([]byte, error) { + b, err := json.Marshal(f) + b = append(b, []byte("\n")...) + return b, err +} diff --git a/exporter/tracking_event_test.go b/exporter/tracking_event_test.go new file mode 100644 index 00000000000..ff1bb6e750c --- /dev/null +++ b/exporter/tracking_event_test.go @@ -0,0 +1,179 @@ +package exporter_test + +import ( + "fmt" + "testing" + "text/template" + + "github.com/stretchr/testify/assert" + "github.com/thomaspoignant/go-feature-flag/exporter" +) + +func TestTrackingEvent_FormatInCSV(t *testing.T) { + tests := []struct { + name string + trackingEvent *exporter.TrackingEvent + template string + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "Should return a marshalled JSON string of the tracking event", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + EvaluationContext: map[string]any{"targetingKey": "ABCD"}, + TrackingDetails: map[string]interface{}{ + "event": "123", + }, + }, + template: `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .EvaluationContext}};{{ .TrackingDetails}}`, + want: `tracking;anonymousUser;ABCD;1617970547;map[targetingKey:ABCD];map[event:123]`, + wantErr: assert.NoError, + }, + { + name: "Should return a marshalled JSON string of the tracking event with evaluation context attributes", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + EvaluationContext: map[string]any{"targetingKey": "ABCD", "toto": 123}, + TrackingDetails: map[string]interface{}{ + "event": "123", + }, + }, + template: `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .EvaluationContext}};{{ .TrackingDetails}}`, + want: `tracking;anonymousUser;ABCD;1617970547;map[targetingKey:ABCD toto:123];map[event:123]`, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + csvTemplate, err := template.New("test").Parse(tt.template) + assert.NoError(t, err) + got, err := tt.trackingEvent.FormatInCSV(csvTemplate) + tt.wantErr(t, err) + if err == nil { + assert.Equal(t, tt.want, string(got)) + } + }) + } +} + +func TestTrackingEvent_FormatInJSON(t *testing.T) { + tests := []struct { + name string + trackingEvent *exporter.TrackingEvent + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "Should return a marshalled JSON string of the tracking event", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + EvaluationContext: map[string]any{"targetingKey": "ABCD"}, + TrackingDetails: map[string]interface{}{ + "event": "123", + }, + }, + want: `{"kind":"tracking","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","evaluationContext":{"targetingKey":"ABCD"},"trackingEventDetails":{"event":"123"}}`, + wantErr: assert.NoError, + }, + { + name: "Should return a marshalled JSON string of the tracking event with evaluation context attributes", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + EvaluationContext: map[string]any{"targetingKey": "ABCD", "toto": 123}, + TrackingDetails: map[string]interface{}{ + "event": "123", + }, + }, + want: `{"kind":"tracking","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","evaluationContext":{"targetingKey":"ABCD","toto":123},"trackingEventDetails":{"event":"123"}}`, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.trackingEvent.FormatInJSON() + tt.wantErr(t, err) + if err == nil { + fmt.Println(string(got)) + assert.JSONEq(t, tt.want, string(got)) + } + }) + } +} + +func TestTrackingEvent_GetKey(t *testing.T) { + tests := []struct { + name string + trackingEvent *exporter.TrackingEvent + want string + }{ + { + name: "return existing key", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + }, + want: "random-key", + }, + { + name: "empty key", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "", + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.trackingEvent.GetKey()) + }) + } +} + +func TestTrackingEvent_GetUserKey(t *testing.T) { + tests := []struct { + name string + trackingEvent *exporter.TrackingEvent + want string + }{ + { + name: "return existing key", + trackingEvent: &exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + }, + want: "ABCD", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.trackingEvent.GetUserKey()) + }) + } +} diff --git a/exporter/webhookexporter/exporter.go b/exporter/webhookexporter/exporter.go index 615575edd1b..fce9be62fe5 100644 --- a/exporter/webhookexporter/exporter.go +++ b/exporter/webhookexporter/exporter.go @@ -56,14 +56,14 @@ type webhookPayload struct { Meta map[string]string `json:"meta"` // events is the list of the event we send in the payload - Events []exporter.FeatureEvent `json:"events"` + Events []exporter.ExportableEvent `json:"events"` } // Export is sending a collection of events in a webhook call. func (f *Exporter) Export( ctx context.Context, _ *fflog.FFLogger, - featureEvents []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { f.init.Do(func() { if f.httpClient == nil { @@ -82,7 +82,7 @@ func (f *Exporter) Export( body := webhookPayload{ Meta: f.Meta, - Events: featureEvents, + Events: events, } payload, err := json.Marshal(body) if err != nil { diff --git a/exporter/webhookexporter/exporter_test.go b/exporter/webhookexporter/exporter_test.go index 3f8783d1dc5..86b46907801 100644 --- a/exporter/webhookexporter/exporter_test.go +++ b/exporter/webhookexporter/exporter_test.go @@ -14,7 +14,7 @@ import ( func TestWebhook_IsBulk(t *testing.T) { exporter := Exporter{} - assert.True(t, exporter.IsBulk(), "DeprecatedExporter is a bulk exporter") + assert.True(t, exporter.IsBulk(), "DeprecatedExporterV1 is a bulk exporter") } func TestWebhook_Export(t *testing.T) { @@ -28,7 +28,7 @@ func TestWebhook_Export(t *testing.T) { } type args struct { logger *fflog.FFLogger - featureEvents []exporter.FeatureEvent + featureEvents []exporter.ExportableEvent } type expected struct { bodyFilePath string @@ -58,12 +58,12 @@ func TestWebhook_Export(t *testing.T) { }, args: args{ logger: logger, - featureEvents: []exporter.FeatureEvent{ - { + featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -85,12 +85,12 @@ func TestWebhook_Export(t *testing.T) { }, args: args{ logger: logger, - featureEvents: []exporter.FeatureEvent{ - { + featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -112,12 +112,12 @@ func TestWebhook_Export(t *testing.T) { }, args: args{ logger: logger, - featureEvents: []exporter.FeatureEvent{ - { + featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -135,12 +135,12 @@ func TestWebhook_Export(t *testing.T) { }, args: args{ logger: logger, - featureEvents: []exporter.FeatureEvent{ - { + featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Source: "SERVER", }, @@ -158,12 +158,12 @@ func TestWebhook_Export(t *testing.T) { }, args: args{ logger: logger, - featureEvents: []exporter.FeatureEvent{ - { + featureEvents: []exporter.ExportableEvent{ + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", Variation: "Default", Value: "YO", Default: false, Source: "SERVER", }, - { + exporter.FeatureEvent{ Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", Variation: "Default", Value: "YO2", Default: false, Version: "127", Source: "SERVER", }, @@ -220,7 +220,7 @@ func TestWebhook_Export_impossibleToParse(t *testing.T) { err := f.Export( context.Background(), &fflog.FFLogger{LeveledLogger: slog.Default()}, - []exporter.FeatureEvent{}, + []exporter.ExportableEvent{}, ) assert.EqualError( t, diff --git a/feature_flag.go b/feature_flag.go index 8bd3aacb2e0..c35747741e7 100644 --- a/feature_flag.go +++ b/feature_flag.go @@ -11,6 +11,7 @@ import ( "github.com/thomaspoignant/go-feature-flag/exporter" "github.com/thomaspoignant/go-feature-flag/internal/cache" + "github.com/thomaspoignant/go-feature-flag/internal/notification" "github.com/thomaspoignant/go-feature-flag/model/dto" "github.com/thomaspoignant/go-feature-flag/notifier/logsnotifier" "github.com/thomaspoignant/go-feature-flag/retriever" @@ -43,11 +44,12 @@ func Init(config Config) error { // GoFeatureFlag is the main object of the library // it contains the cache, the config, the updater and the exporter. type GoFeatureFlag struct { - cache cache.Manager - config Config - bgUpdater backgroundUpdater - featureEventDataExporter exporter.Manager[exporter.FeatureEvent] - retrieverManager *retriever.Manager + cache cache.Manager + config Config + bgUpdater backgroundUpdater + featureEventDataExporter exporter.Manager[exporter.FeatureEvent] + trackingEventDataExporter exporter.Manager[exporter.TrackingEvent] + retrieverManager *retriever.Manager // evalExporterWg is a wait group to wait for the evaluation exporter to finish the export before closing GOFF evalExporterWg sync.WaitGroup } @@ -112,7 +114,7 @@ func New(config Config) (*GoFeatureFlag, error) { go goFF.startFlagUpdaterDaemon() } - goFF.featureEventDataExporter = + goFF.featureEventDataExporter, goFF.trackingEventDataExporter = initializeDataExporters(config, goFF.config.internalLogger) config.internalLogger.Debug("GO Feature Flag is initialized") return goFF, nil @@ -134,10 +136,10 @@ func adjustPollingInterval(pollingInterval time.Duration) time.Duration { } // initializeNotificationService is a function that will initialize the notification service with the notifiers -func initializeNotificationService(config Config) cache.Service { +func initializeNotificationService(config Config) notification.Service { notifiers := config.Notifiers notifiers = append(notifiers, &logsnotifier.Notifier{Logger: config.internalLogger}) - return cache.NewNotificationService(notifiers) + return notification.NewService(notifiers) } // initializeRetrieverManager is a function that will initialize the retriever manager with the retrievers @@ -151,12 +153,11 @@ func initializeRetrieverManager(config Config) (*retriever.Manager, error) { return manager, err } -func initializeDataExporters( - config Config, - logger *fflog.FFLogger, -) exporter.Manager[exporter.FeatureEvent] { +func initializeDataExporters(config Config, logger *fflog.FFLogger) ( + exporter.Manager[exporter.FeatureEvent], exporter.Manager[exporter.TrackingEvent]) { exporters := config.GetDataExporters() featureEventExporterConfigs := make([]exporter.Config, 0) + trackingEventExporterConfigs := make([]exporter.Config, 0) if len(exporters) > 0 { for _, exp := range exporters { c := exporter.Config{ @@ -164,17 +165,28 @@ func initializeDataExporters( FlushInterval: exp.FlushInterval, MaxEventInMemory: exp.MaxEventInMemory, } + if exp.ExporterEventType == TrackingEventExporter { + trackingEventExporterConfigs = append(trackingEventExporterConfigs, c) + continue + } featureEventExporterConfigs = append(featureEventExporterConfigs, c) } } + var trackingEventManager exporter.Manager[exporter.TrackingEvent] + if len(trackingEventExporterConfigs) > 0 { + trackingEventManager = exporter.NewManager[exporter.TrackingEvent]( + config.Context, trackingEventExporterConfigs, config.ExporterCleanQueueInterval, logger) + trackingEventManager.Start() + } + var featureEventManager exporter.Manager[exporter.FeatureEvent] if len(featureEventExporterConfigs) > 0 { featureEventManager = exporter.NewManager[exporter.FeatureEvent]( config.Context, featureEventExporterConfigs, config.ExporterCleanQueueInterval, logger) featureEventManager.Start() } - return featureEventManager + return featureEventManager, trackingEventManager } // handleFirstRetrieverError is a function that will handle the first error when trying to retrieve @@ -255,6 +267,10 @@ func (g *GoFeatureFlag) Close() { g.featureEventDataExporter.Stop() } + if g.trackingEventDataExporter != nil { + g.trackingEventDataExporter.Stop() + } + if g.retrieverManager != nil { _ = g.retrieverManager.Shutdown(g.config.Context) } diff --git a/feature_flag_test.go b/feature_flag_test.go index 23968d1fbe5..368db880bbc 100644 --- a/feature_flag_test.go +++ b/feature_flag_test.go @@ -85,16 +85,23 @@ func TestStartWithMinInterval(t *testing.T) { } func TestValidUseCase(t *testing.T) { + cliExport := mock.Exporter{Bulk: false} // Valid use case err := ffclient.Init(ffclient.Config{ PollingInterval: 5 * time.Second, Retriever: &fileretriever.Retriever{Path: "testdata/flag-config.yaml"}, LeveledLogger: slog.Default(), - DataExporter: ffclient.DataExporter{ - FlushInterval: 10 * time.Second, - MaxEventInMemory: 1000, - Exporter: &mock.Exporter{ - Bulk: true, + DataExporters: []ffclient.DataExporter{ + { + FlushInterval: 10 * time.Second, + MaxEventInMemory: 1000, + Exporter: &mock.Exporter{ + Bulk: true, + }, + }, + { + Exporter: &cliExport, + ExporterEventType: ffclient.TrackingEventExporter, }, }, }) @@ -117,6 +124,8 @@ func TestValidUseCase(t *testing.T) { ffclient.SetOffline(false) assert.False(t, ffclient.IsOffline()) assert.True(t, ffclient.ForceRefresh()) + ffclient.Track("toto", user, map[string]interface{}{"key": "value"}) + assert.Equal(t, 1, len(cliExport.ExportedEvents)) } func TestValidUseCaseToml(t *testing.T) { diff --git a/ffcontext/context.go b/ffcontext/context.go index 58683e8e33b..f0694098803 100644 --- a/ffcontext/context.go +++ b/ffcontext/context.go @@ -1,33 +1,35 @@ package ffcontext +import "encoding/json" + type Context interface { - // GetKey return the unique key for the context. + // GetKey return the unique targetingKey for the context. GetKey() string // IsAnonymous return if the context is about an anonymous user or not. IsAnonymous() bool - // GetCustom return all the custom properties added to the context. + // GetCustom return all the attributes properties added to the context. GetCustom() map[string]interface{} - // AddCustomAttribute allows to add a custom attribute into the context. + // AddCustomAttribute allows to add a attributes attribute into the context. AddCustomAttribute(name string, value interface{}) // ExtractGOFFProtectedFields extract the goff specific attributes from the evaluation context. ExtractGOFFProtectedFields() GoffContextSpecifics } -// value is a type to define custom attribute. +// value is a type to define attributes. type value map[string]interface{} -// NewEvaluationContext creates a new evaluation context identified by the given key. +// NewEvaluationContext creates a new evaluation context identified by the given targetingKey. func NewEvaluationContext(key string) EvaluationContext { - return EvaluationContext{key: key, custom: map[string]interface{}{}} + return EvaluationContext{targetingKey: key, attributes: map[string]interface{}{}} } // Deprecated: NewAnonymousEvaluationContext is here for compatibility reason. -// Please use NewEvaluationContext instead and add a custom attribute to know that it is an anonymous user. +// Please use NewEvaluationContext instead and add a attributes attribute to know that it is an anonymous user. // -// ctx := NewEvaluationContext("my-key") +// ctx := NewEvaluationContext("my-targetingKey") // ctx.AddCustomAttribute("anonymous", true) func NewAnonymousEvaluationContext(key string) EvaluationContext { - return EvaluationContext{key: key, custom: map[string]interface{}{ + return EvaluationContext{targetingKey: key, attributes: map[string]interface{}{ "anonymous": true, }} } @@ -42,18 +44,31 @@ func NewAnonymousEvaluationContext(key string) EvaluationContext { // To construct an EvaluationContext, use either a simple constructor (NewEvaluationContext) or the builder pattern // with NewEvaluationContextBuilder. type EvaluationContext struct { - key string // only mandatory attribute - custom value + // uniquely identifying the subject (end-user, or client service) of a flag evaluation + targetingKey string + attributes value +} + +// MarshalJSON is a custom JSON marshaller for EvaluationContext. +// It will only marshal the targetingKey and the attributes of the context and avoid to expose the internal structure. +func (u EvaluationContext) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + TargetingKey string `json:"targetingKey"` + Attributes value `json:"attributes"` + }{ + TargetingKey: u.targetingKey, + Attributes: u.attributes, + }) } -// GetKey return the unique key for the user. +// GetKey return the unique targetingKey for the user. func (u EvaluationContext) GetKey() string { - return u.key + return u.targetingKey } // IsAnonymous return if the user is anonymous or not. func (u EvaluationContext) IsAnonymous() bool { - anonymous := u.custom["anonymous"] + anonymous := u.attributes["anonymous"] switch v := anonymous.(type) { case bool: return v @@ -62,22 +77,28 @@ func (u EvaluationContext) IsAnonymous() bool { } } -// GetCustom return all the custom properties of a user. +// GetCustom return all the attributes properties of a user. func (u EvaluationContext) GetCustom() map[string]interface{} { - return u.custom + return u.attributes } -// AddCustomAttribute allows to add a custom attribute into the user. +// AddCustomAttribute allows to add a attributes attribute into the user. func (u EvaluationContext) AddCustomAttribute(name string, value interface{}) { if name != "" { - u.custom[name] = value + u.attributes[name] = value } } +func (u EvaluationContext) ToMap() map[string]any { + resMap := u.attributes + resMap["targetingKey"] = u.targetingKey + return resMap +} + // ExtractGOFFProtectedFields extract the goff specific attributes from the evaluation context. func (u EvaluationContext) ExtractGOFFProtectedFields() GoffContextSpecifics { goff := GoffContextSpecifics{} - switch v := u.custom["gofeatureflag"].(type) { + switch v := u.attributes["gofeatureflag"].(type) { case map[string]string: goff.addCurrentDateTime(v["currentDateTime"]) goff.addListFlags(v["flagList"]) diff --git a/ffcontext/context_builder.go b/ffcontext/context_builder.go index 7ff0ea5c480..71183ee839d 100644 --- a/ffcontext/context_builder.go +++ b/ffcontext/context_builder.go @@ -1,8 +1,8 @@ package ffcontext -// NewEvaluationContextBuilder constructs a new EvaluationContextBuilder, specifying the user key. +// NewEvaluationContextBuilder constructs a new EvaluationContextBuilder, specifying the user targetingKey. // -// For authenticated users, the key may be a username or e-mail address. For anonymous users, +// For authenticated users, the targetingKey may be a username or e-mail address. For anonymous users, // this could be an IP address or session ID. func NewEvaluationContextBuilder(key string) EvaluationContextBuilder { return &evaluationContextBuilderImpl{ @@ -36,7 +36,7 @@ func (u *evaluationContextBuilderImpl) Anonymous(anonymous bool) EvaluationConte return u } -// AddCustom allows you to add a custom attribute to the EvaluationContext. +// AddCustom allows you to add an attributes attribute to the EvaluationContext. func (u *evaluationContextBuilderImpl) AddCustom( key string, value interface{}, @@ -48,7 +48,7 @@ func (u *evaluationContextBuilderImpl) AddCustom( // Build is creating the EvaluationContext. func (u *evaluationContextBuilderImpl) Build() EvaluationContext { return EvaluationContext{ - key: u.key, - custom: u.custom, + targetingKey: u.key, + attributes: u.custom, } } diff --git a/ffcontext/context_builder_test.go b/ffcontext/context_builder_test.go index 43c29d6bff9..3b69dbf4440 100644 --- a/ffcontext/context_builder_test.go +++ b/ffcontext/context_builder_test.go @@ -13,71 +13,71 @@ func TestNewUser(t *testing.T) { want EvaluationContext }{ { - name: "Builder with only key", - got: NewEvaluationContextBuilder("random-key").Build(), + name: "Builder with only targetingKey", + got: NewEvaluationContextBuilder("random-targetingKey").Build(), want: EvaluationContext{ - key: "random-key", - custom: map[string]interface{}{}, + targetingKey: "random-targetingKey", + attributes: map[string]interface{}{}, }, }, { - name: "Builder with custom attribute", - got: NewEvaluationContextBuilder("random-key"). - AddCustom("test", "custom"). + name: "Builder with attributes attribute", + got: NewEvaluationContextBuilder("random-targetingKey"). + AddCustom("test", "attributes"). Build(), want: EvaluationContext{ - key: "random-key", - custom: map[string]interface{}{ - "test": "custom", + targetingKey: "random-targetingKey", + attributes: map[string]interface{}{ + "test": "attributes", }, }, }, { - name: "Builder with custom attribute", - got: NewEvaluationContextBuilder("random-key"). + name: "Builder with attributes attribute", + got: NewEvaluationContextBuilder("random-targetingKey"). Anonymous(true). - AddCustom("test", "custom"). + AddCustom("test", "attributes"). Build(), want: EvaluationContext{ - key: "random-key", - custom: map[string]interface{}{ - "test": "custom", + targetingKey: "random-targetingKey", + attributes: map[string]interface{}{ + "test": "attributes", "anonymous": true, }, }, }, { - name: "NewUser with key", - got: NewEvaluationContext("random-key"), + name: "NewUser with targetingKey", + got: NewEvaluationContext("random-targetingKey"), want: EvaluationContext{ - key: "random-key", - custom: map[string]interface{}{}, + targetingKey: "random-targetingKey", + attributes: map[string]interface{}{}, }, }, { - name: "NewUser without key", + name: "NewUser without targetingKey", got: NewEvaluationContext(""), want: EvaluationContext{ - key: "", - custom: map[string]interface{}{}, + targetingKey: "", + attributes: map[string]interface{}{}, }, }, { - name: "NewAnonymousUser with key", - got: NewAnonymousEvaluationContext("random-key"), + name: "NewAnonymousUser with targetingKey", + got: NewAnonymousEvaluationContext("random-targetingKey"), want: EvaluationContext{ - key: "random-key", - custom: map[string]interface{}{ + targetingKey: "random-targetingKey", + attributes: map[string]interface{}{ "anonymous": true, }, }, }, { - name: "NewAnonymousUser without key", + name: "NewAnonymousUser without targetingKey", got: NewAnonymousEvaluationContext(""), want: EvaluationContext{ - key: "", - custom: map[string]interface{}{ + targetingKey: "", + attributes: map[string]interface{}{ "anonymous": true, }, }, diff --git a/ffcontext/context_test.go b/ffcontext/context_test.go index 30dbd7ab409..cb982496039 100644 --- a/ffcontext/context_test.go +++ b/ffcontext/context_test.go @@ -54,7 +54,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }{ { name: "context goff specifics as map[string]string", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]string{ "currentDateTime": time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC). Format(time.RFC3339), @@ -66,7 +66,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics as map[string]interface and date as time.Time", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]interface{}{ "currentDateTime": time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC), }). @@ -77,7 +77,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics as map[string]interface and date as *time.Time", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]interface{}{ "currentDateTime": testconvert.Time( time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC), @@ -90,7 +90,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics as map[string]interface", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]interface{}{ "currentDateTime": time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC). Format(time.RFC3339), @@ -102,7 +102,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics nil", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", nil). Build(), want: ffcontext.GoffContextSpecifics{ @@ -111,14 +111,14 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "no context goff specifics", - ctx: ffcontext.NewEvaluationContextBuilder("my-key").Build(), + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey").Build(), want: ffcontext.GoffContextSpecifics{ CurrentDateTime: nil, }, }, { name: "context goff specifics as GoffContextSpecifics type", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", ffcontext.GoffContextSpecifics{ CurrentDateTime: testconvert.Time(time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC)), }). @@ -129,7 +129,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics as GoffContextSpecifics type contains flagList", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", ffcontext.GoffContextSpecifics{ CurrentDateTime: testconvert.Time(time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC)), FlagList: []string{"flag1", "flag2"}, @@ -142,7 +142,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics as map[string]interface type contains flagList", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]interface{}{ "currentDateTime": testconvert.Time(time.Date(2022, 8, 1, 0, 0, 10, 0, time.UTC)). Format(time.RFC3339), @@ -156,7 +156,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics only flagList", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]interface{}{ "flagList": []string{"flag1", "flag2"}, }). @@ -167,7 +167,7 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }, { name: "context goff specifics with exporter metadata", - ctx: ffcontext.NewEvaluationContextBuilder("my-key"). + ctx: ffcontext.NewEvaluationContextBuilder("my-targetingKey"). AddCustom("gofeatureflag", map[string]interface{}{ "exporterMetadata": map[string]interface{}{ "toto": 123, @@ -195,3 +195,92 @@ func Test_ExtractGOFFProtectedFields(t *testing.T) { }) } } + +func TestEvaluationContext_MarshalJSON(t *testing.T) { + tests := []struct { + name string + context ffcontext.EvaluationContext + expected string + }{ + { + name: "marshal with empty attributes", + context: ffcontext.NewEvaluationContext("test-key"), + expected: `{"targetingKey":"test-key","attributes":{}}`, + }, + { + name: "marshal with attributes", + context: ffcontext.NewEvaluationContextBuilder("test-key"). + AddCustom("attr1", "value1"). + AddCustom("attr2", 123). + Build(), + expected: `{"targetingKey":"test-key","attributes":{"attr1":"value1","attr2":123}}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := tt.context.MarshalJSON() + assert.NoError(t, err) + assert.JSONEq(t, tt.expected, string(data)) + }) + } +} + +func TestEvaluationContext_ToMap(t *testing.T) { + tests := []struct { + name string + context ffcontext.EvaluationContext + expected map[string]interface{} + }{ + { + name: "empty attributes", + context: ffcontext.NewEvaluationContext("test-key"), + expected: map[string]interface{}{"targetingKey": "test-key"}, + }, + { + name: "attributes with values", + context: ffcontext.NewEvaluationContextBuilder("test-key"). + AddCustom("attr1", "value1"). + AddCustom("attr2", 123). + Build(), + expected: map[string]interface{}{ + "targetingKey": "test-key", + "attr1": "value1", + "attr2": 123, + }, + }, + { + name: "attributes with nested map", + context: ffcontext.NewEvaluationContextBuilder("test-key"). + AddCustom("nested", map[string]interface{}{ + "key1": "value1", + "key2": 42, + }). + Build(), + expected: map[string]interface{}{ + "targetingKey": "test-key", + "nested": map[string]interface{}{ + "key1": "value1", + "key2": 42, + }, + }, + }, + { + name: "attributes with nil value", + context: ffcontext.NewEvaluationContextBuilder("test-key"). + AddCustom("attr1", nil). + Build(), + expected: map[string]interface{}{ + "targetingKey": "test-key", + "attr1": nil, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.context.ToMap() + assert.Equal(t, tt.expected, got) + }) + } +} diff --git a/go.mod b/go.mod index 28c7899bfa9..695a65cb581 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 github.com/diegoholiveira/jsonlogic/v3 v3.8.1 github.com/fsouza/fake-gcs-server v1.52.2 + github.com/go-viper/mapstructure/v2 v2.2.1 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 @@ -149,7 +150,6 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/internal/cache/cache_manager.go b/internal/cache/cache_manager.go index 31838717fe9..c9118e31e73 100644 --- a/internal/cache/cache_manager.go +++ b/internal/cache/cache_manager.go @@ -12,6 +12,7 @@ import ( "github.com/BurntSushi/toml" "github.com/google/go-cmp/cmp" "github.com/thomaspoignant/go-feature-flag/internal/flag" + "github.com/thomaspoignant/go-feature-flag/internal/notification" "github.com/thomaspoignant/go-feature-flag/model/dto" "github.com/thomaspoignant/go-feature-flag/utils/fflog" "gopkg.in/yaml.v3" @@ -29,14 +30,14 @@ type Manager interface { type cacheManagerImpl struct { inMemoryCache Cache mutex sync.RWMutex - notificationService Service + notificationService notification.Service latestUpdate time.Time logger *fflog.FFLogger persistentFlagConfigurationFile string } func New( - notificationService Service, + notificationService notification.Service, persistentFlagConfigurationFile string, logger *fflog.FFLogger, ) Manager { diff --git a/internal/cache/cache_manager_test.go b/internal/cache/cache_manager_test.go index ad0c8da45f4..79a94777390 100644 --- a/internal/cache/cache_manager_test.go +++ b/internal/cache/cache_manager_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/thomaspoignant/go-feature-flag/internal/cache" "github.com/thomaspoignant/go-feature-flag/internal/flag" + "github.com/thomaspoignant/go-feature-flag/internal/notification" "github.com/thomaspoignant/go-feature-flag/model/dto" "github.com/thomaspoignant/go-feature-flag/notifier" "github.com/thomaspoignant/go-feature-flag/testutils/mock" @@ -248,7 +249,7 @@ variation = "false_var" for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fCache := cache.New(cache.NewNotificationService([]notifier.Notifier{}), "", + fCache := cache.New(notification.NewService([]notifier.Notifier{}), "", &fflog.FFLogger{LeveledLogger: slog.Default()}) newFlags, err := fCache.ConvertToFlagStruct(tt.args.loadedFlags, tt.flagFormat) if tt.wantErr { @@ -412,7 +413,7 @@ test-flag2: for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fCache := cache.New(cache.NewNotificationService([]notifier.Notifier{}), "", nil) + fCache := cache.New(notification.NewService([]notifier.Notifier{}), "", nil) newFlags, err := fCache.ConvertToFlagStruct(tt.args.loadedFlags, tt.flagFormat) if tt.wantErr { assert.Error(t, err) @@ -453,7 +454,7 @@ func Test_cacheManagerImpl_GetLatestUpdateDate(t *testing.T) { trackEvents: false `) - fCache := cache.New(cache.NewNotificationService([]notifier.Notifier{}), "", nil) + fCache := cache.New(notification.NewService([]notifier.Notifier{}), "", nil) timeBefore := fCache.GetLatestUpdateDate() newFlags, _ := fCache.ConvertToFlagStruct(loadedFlags, "yaml") _ = fCache.UpdateCache(newFlags, &fflog.FFLogger{LeveledLogger: slog.Default()}, true) @@ -485,7 +486,7 @@ func Test_persistCacheAndRestartCacheWithIt(t *testing.T) { err = yaml.Unmarshal(loadedFlags, &loadedFlagsMap) assert.NoError(t, err) - fCache := cache.New(cache.NewNotificationService([]notifier.Notifier{}), file.Name(), nil) + fCache := cache.New(notification.NewService([]notifier.Notifier{}), file.Name(), nil) err = fCache.UpdateCache(loadedFlagsMap, &fflog.FFLogger{LeveledLogger: slog.Default()}, true) assert.NoError(t, err) allFlags1, err := fCache.AllFlags() @@ -494,7 +495,7 @@ func Test_persistCacheAndRestartCacheWithIt(t *testing.T) { time.Sleep(100 * time.Millisecond) // waiting to let the go routine write in the file // we start a new cache with the file persisted - fCache2 := cache.New(cache.NewNotificationService([]notifier.Notifier{}), "", nil) + fCache2 := cache.New(notification.NewService([]notifier.Notifier{}), "", nil) content, err := os.ReadFile(file.Name()) assert.NoError(t, err) loadedFlagsMap2 := map[string]dto.DTO{} diff --git a/internal/cache/notification_service.go b/internal/notification/notification_service.go similarity index 95% rename from internal/cache/notification_service.go rename to internal/notification/notification_service.go index 2e167b055b7..0fe915b5466 100644 --- a/internal/cache/notification_service.go +++ b/internal/notification/notification_service.go @@ -1,4 +1,4 @@ -package cache +package notification import ( "log/slog" @@ -15,7 +15,7 @@ type Service interface { Notify(oldCache map[string]flag.Flag, newCache map[string]flag.Flag, log *fflog.FFLogger) } -func NewNotificationService(notifiers []notifier.Notifier) Service { +func NewService(notifiers []notifier.Notifier) Service { return ¬ificationService{ Notifiers: notifiers, waitGroup: &sync.WaitGroup{}, diff --git a/internal/cache/notification_service_priv_test.go b/internal/notification/notification_service_priv_test.go similarity index 88% rename from internal/cache/notification_service_priv_test.go rename to internal/notification/notification_service_priv_test.go index cbb4930e176..764f56be784 100644 --- a/internal/cache/notification_service_priv_test.go +++ b/internal/notification/notification_service_priv_test.go @@ -1,6 +1,6 @@ //go:build !race -package cache_test +package notification_test import ( "fmt" @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/thejerf/slogassert" - "github.com/thomaspoignant/go-feature-flag/internal/cache" "github.com/thomaspoignant/go-feature-flag/internal/flag" + "github.com/thomaspoignant/go-feature-flag/internal/notification" "github.com/thomaspoignant/go-feature-flag/notifier" "github.com/thomaspoignant/go-feature-flag/testutils/testconvert" "github.com/thomaspoignant/go-feature-flag/utils/fflog" @@ -19,7 +19,7 @@ import ( func Test_notificationService_callNotifier(t *testing.T) { n := &NotifierMock{} - c := cache.NewNotificationService([]notifier.Notifier{n}) + c := notification.NewService([]notifier.Notifier{n}) oldCache := map[string]flag.Flag{ "yo": &flag.InternalFlag{Version: testconvert.String("1.0")}, } @@ -33,7 +33,7 @@ func Test_notificationService_callNotifier(t *testing.T) { func Test_notificationService_no_difference(t *testing.T) { n := &NotifierMock{} - c := cache.NewNotificationService([]notifier.Notifier{n}) + c := notification.NewService([]notifier.Notifier{n}) oldCache := map[string]flag.Flag{ "yo": &flag.InternalFlag{Version: testconvert.String("1.0")}, } @@ -49,7 +49,7 @@ func Test_notificationService_with_error(t *testing.T) { handler := slogassert.New(t, slog.LevelDebug, nil) logger := slog.New(handler) n := &NotifierMock{WithError: true} - c := cache.NewNotificationService([]notifier.Notifier{n}) + c := notification.NewService([]notifier.Notifier{n}) oldCache := map[string]flag.Flag{ "yo": &flag.InternalFlag{Version: testconvert.String("1.0")}, } diff --git a/internal/cache/notification_service_test.go b/internal/notification/notification_service_test.go similarity index 99% rename from internal/cache/notification_service_test.go rename to internal/notification/notification_service_test.go index 83e71878f9a..2b68b8374c7 100644 --- a/internal/cache/notification_service_test.go +++ b/internal/notification/notification_service_test.go @@ -1,4 +1,4 @@ -package cache +package notification import ( "sync" diff --git a/testutils/exporter.go b/testutils/exporter.go new file mode 100644 index 00000000000..35409714043 --- /dev/null +++ b/testutils/exporter.go @@ -0,0 +1,31 @@ +package testutils + +import "text/template" + +func NewExportableMockEvent(name string) ExportableMockEvent { + return ExportableMockEvent{name: name} +} + +type ExportableMockEvent struct { + name string +} + +func (e ExportableMockEvent) GetUserKey() string { + return e.name +} + +func (e ExportableMockEvent) GetKey() string { + return e.name +} + +func (e ExportableMockEvent) GetCreationDate() int64 { + return 0 +} + +func (e ExportableMockEvent) FormatInCSV(_ *template.Template) ([]byte, error) { + return []byte(e.name), nil +} + +func (e ExportableMockEvent) FormatInJSON() ([]byte, error) { + return []byte(`{"name":"` + e.name + `"}`), nil +} diff --git a/testutils/mock/event_store.go b/testutils/mock/event_store.go index a53b823122f..788f8a8f0c4 100644 --- a/testutils/mock/event_store.go +++ b/testutils/mock/event_store.go @@ -9,11 +9,11 @@ import ( const consumerNameError = "error" -type implMockEventStore[T any] struct { +type implMockEventStore[T exporter.ExportableEvent] struct { store []T } -func NewEventStore[T any]() exporter.EventStore[T] { +func NewEventStore[T exporter.ExportableEvent]() exporter.EventStore[T] { store := &implMockEventStore[T]{} return store } diff --git a/testutils/mock/exporter_mock.go b/testutils/mock/exporter_mock.go index b85c6659429..26b1c28937a 100644 --- a/testutils/mock/exporter_mock.go +++ b/testutils/mock/exporter_mock.go @@ -11,10 +11,10 @@ import ( type ExporterMock interface { exporter.CommonExporter - GetExportedEvents() []exporter.FeatureEvent + GetExportedEvents() []exporter.ExportableEvent } type Exporter struct { - ExportedEvents []exporter.FeatureEvent + ExportedEvents []exporter.ExportableEvent Err error ExpectedNumberErr int CurrentNumberErr int @@ -27,7 +27,7 @@ type Exporter struct { func (m *Exporter) Export( _ context.Context, _ *fflog.FFLogger, - events []exporter.FeatureEvent, + events []exporter.ExportableEvent, ) error { m.once.Do(m.initMutex) m.mutex.Lock() @@ -42,7 +42,7 @@ func (m *Exporter) Export( return nil } -func (m *Exporter) GetExportedEvents() []exporter.FeatureEvent { +func (m *Exporter) GetExportedEvents() []exporter.ExportableEvent { m.once.Do(m.initMutex) m.mutex.Lock() defer m.mutex.Unlock() @@ -87,11 +87,16 @@ func (m *ExporterDeprecated) Export( return nil } -func (m *ExporterDeprecated) GetExportedEvents() []exporter.FeatureEvent { +func (m *ExporterDeprecated) GetExportedEvents() []exporter.ExportableEvent { m.once.Do(m.initMutex) m.mutex.Lock() defer m.mutex.Unlock() - return m.ExportedEvents + + exportableEvents := make([]exporter.ExportableEvent, len(m.ExportedEvents)) + for index, event := range m.ExportedEvents { + exportableEvents[index] = event + } + return exportableEvents } func (m *ExporterDeprecated) IsBulk() bool { @@ -101,3 +106,53 @@ func (m *ExporterDeprecated) IsBulk() bool { func (m *ExporterDeprecated) initMutex() { m.mutex = sync.Mutex{} } + +// ExporterDeprecatedV2 ----- +type ExporterDeprecatedV2 struct { + ExportedEvents []exporter.FeatureEvent + Err error + ExpectedNumberErr int + CurrentNumberErr int + Bulk bool + + mutex sync.Mutex + once sync.Once +} + +func (m *ExporterDeprecatedV2) Export( + _ context.Context, + _ *fflog.FFLogger, + events []exporter.FeatureEvent, +) error { + m.once.Do(m.initMutex) + m.mutex.Lock() + defer m.mutex.Unlock() + m.ExportedEvents = append(m.ExportedEvents, events...) + if m.Err != nil { + if m.ExpectedNumberErr > m.CurrentNumberErr { + m.CurrentNumberErr++ + return m.Err + } + } + return nil +} + +func (m *ExporterDeprecatedV2) GetExportedEvents() []exporter.ExportableEvent { + m.once.Do(m.initMutex) + m.mutex.Lock() + defer m.mutex.Unlock() + + exportableEvents := make([]exporter.ExportableEvent, len(m.ExportedEvents)) + for index, event := range m.ExportedEvents { + exportableEvents[index] = event + } + return exportableEvents +} + +func (m *ExporterDeprecatedV2) IsBulk() bool { + return m.Bulk +} + +func (m *ExporterDeprecatedV2) initMutex() { + m.mutex = sync.Mutex{} +} diff --git a/testutils/mock/tracking_event_exporter_mock.go b/testutils/mock/tracking_event_exporter_mock.go new file mode 100644 index 00000000000..122bf59621e --- /dev/null +++ b/testutils/mock/tracking_event_exporter_mock.go @@ -0,0 +1,68 @@ +package mock + +import ( + "context" + "sync" + + "github.com/thomaspoignant/go-feature-flag/exporter" + "github.com/thomaspoignant/go-feature-flag/utils/fflog" +) + +type TrackingEventExporter struct { + ExportedEvents []exporter.TrackingEvent + Err error + ExpectedNumberErr int + CurrentNumberErr int + Bulk bool + + mutex sync.Mutex + once sync.Once +} + +func (m *TrackingEventExporter) Export( + _ context.Context, + _ *fflog.FFLogger, + events []exporter.ExportableEvent, +) error { + m.once.Do(m.initMutex) + m.mutex.Lock() + defer m.mutex.Unlock() + switch events := any(events).(type) { + case []exporter.ExportableEvent: + t := make([]exporter.TrackingEvent, len(events)) + for i, v := range events { + t[i] = v.(exporter.TrackingEvent) + } + m.ExportedEvents = append(m.ExportedEvents, t...) + break + case []exporter.TrackingEvent: + m.ExportedEvents = append(m.ExportedEvents, events...) + break + } + if m.Err != nil { + if m.ExpectedNumberErr > m.CurrentNumberErr { + m.CurrentNumberErr++ + return m.Err + } + } + return nil +} + +func (m *TrackingEventExporter) GetExportedEvents() []exporter.ExportableEvent { + m.once.Do(m.initMutex) + m.mutex.Lock() + defer m.mutex.Unlock() + trackingEvents := make([]exporter.ExportableEvent, 0) + for _, event := range m.ExportedEvents { + trackingEvents = append(trackingEvents, event) + } + return trackingEvents +} + +func (m *TrackingEventExporter) IsBulk() bool { + return m.Bulk +} + +func (m *TrackingEventExporter) initMutex() { + m.mutex = sync.Mutex{} +} diff --git a/tracking.go b/tracking.go new file mode 100644 index 00000000000..e6876811342 --- /dev/null +++ b/tracking.go @@ -0,0 +1,42 @@ +package ffclient + +import ( + "time" + + "github.com/thomaspoignant/go-feature-flag/exporter" + "github.com/thomaspoignant/go-feature-flag/ffcontext" +) + +// Track is used to track an event. +// Note: Use this function only if you are using multiple go-feature-flag instances. +func (g *GoFeatureFlag) Track( + trackingEventName string, + ctx ffcontext.EvaluationContext, + trackingEventDetails exporter.TrackingEventDetails, +) { + if g != nil && g.trackingEventDataExporter != nil { + contextKind := "user" + if ctx.IsAnonymous() { + contextKind = "anonymousUser" + } + event := exporter.TrackingEvent{ + Kind: "tracking", + ContextKind: contextKind, + UserKey: ctx.GetKey(), + CreationDate: time.Now().Unix(), + Key: trackingEventName, + EvaluationContext: ctx.ToMap(), + TrackingDetails: trackingEventDetails, + } + g.trackingEventDataExporter.AddEvent(event) + } +} + +// Track is used to track an event. +func Track( + trackingEventName string, + ctx ffcontext.EvaluationContext, + trackingEventDetails exporter.TrackingEventDetails, +) { + ff.Track(trackingEventName, ctx, trackingEventDetails) +} diff --git a/tracking_test.go b/tracking_test.go new file mode 100644 index 00000000000..486753a15ee --- /dev/null +++ b/tracking_test.go @@ -0,0 +1,56 @@ +package ffclient_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + ffclient "github.com/thomaspoignant/go-feature-flag" + "github.com/thomaspoignant/go-feature-flag/ffcontext" + "github.com/thomaspoignant/go-feature-flag/retriever/fileretriever" + "github.com/thomaspoignant/go-feature-flag/testutils/mock" +) + +func TestValidTrackingEvent(t *testing.T) { + exp := mock.TrackingEventExporter{Bulk: false} + goff, err := ffclient.New(ffclient.Config{ + PollingInterval: 500 * time.Millisecond, + Retriever: &fileretriever.Retriever{Path: "./testdata/flag-config.yaml"}, + DataExporters: []ffclient.DataExporter{ + { + FlushInterval: 100 * time.Millisecond, + MaxEventInMemory: 100, + Exporter: &exp, + ExporterEventType: ffclient.TrackingEventExporter, + }, + }, + }) + assert.NoError(t, err) + + goff.Track( + "my-feature-flag", + ffcontext.NewEvaluationContextBuilder("1668d845-051d-4dd9-907a-7ebe6aa2c9da"). + AddCustom("admin", true). + AddCustom("anonymous", true). + Build(), + map[string]interface{}{"additional data": "value"}, + ) + + assert.Equal(t, 1, len(exp.ExportedEvents)) + assert.Equal(t, "1668d845-051d-4dd9-907a-7ebe6aa2c9da", exp.ExportedEvents[0].UserKey) + assert.Equal(t, "my-feature-flag", exp.ExportedEvents[0].Key) + assert.Equal( + t, + map[string]interface{}{ + "targetingKey": "1668d845-051d-4dd9-907a-7ebe6aa2c9da", + "admin": true, + "anonymous": true, + }, + exp.ExportedEvents[0].EvaluationContext, + ) + assert.Equal( + t, + map[string]interface{}{"additional data": "value"}, + exp.ExportedEvents[0].TrackingDetails, + ) +} diff --git a/variation.go b/variation.go index ca3c8540c5c..c0d480523b8 100644 --- a/variation.go +++ b/variation.go @@ -309,6 +309,14 @@ func (g *GoFeatureFlag) CollectEventData(event exporter.FeatureEvent) { } } +// CollectTrackingEventData is collecting tracking events and sending them to the data exporter to be stored. +func (g *GoFeatureFlag) CollectTrackingEventData(event exporter.TrackingEvent) { + if g != nil && g.featureEventDataExporter != nil { + // Add event in the exporter + g.trackingEventDataExporter.AddEvent(event) + } +} + // notifyVariation is logging the evaluation result for a flag // if no logger is provided in the configuration we are not logging anything. func notifyVariation[T model.JSONType](