Code refactoring and testing

Author: rjtokenring

business/tsextractor/tsextractor.go

@@ -45,6 +45,16 @@ func New(iotcl *iot.Client, logger *logrus.Entry) *TsExtractor {
 	return &TsExtractor{iotcl: iotcl, logger: logger}
 }
 
+func computeTimeAlignment(resolutionSeconds, timeWindowInMinutes int) (time.Time, time.Time) {
+	// Compute time alignment
+	if resolutionSeconds <= 60 {
+		resolutionSeconds = 300 // Align to 5 minutes
+	}
+	to := time.Now().Truncate(time.Duration(resolutionSeconds) * time.Second).UTC()
+	from := to.Add(-time.Duration(timeWindowInMinutes) * time.Minute)
+	return from, to
+}
+
 func (a *TsExtractor) ExportTSToS3(
 	ctx context.Context,
 	timeWindowInMinutes int,
@@ -53,8 +63,7 @@ func (a *TsExtractor) ExportTSToS3(
 	destinationS3Bucket string) error {
 
 	// Truncate time to given resolution
-	to := time.Now().Truncate(time.Duration(resolution) * time.Second).UTC()
-	from := to.Add(-time.Duration(timeWindowInMinutes) * time.Minute)
+	from, to := computeTimeAlignment(resolution, timeWindowInMinutes)
 
 	// Open s3 output writer
 	s3cl, err := s3.NewS3Client(destinationS3Bucket)

business/tsextractor/tsextractor_test.go

@@ -0,0 +1,37 @@
+package tsextractor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTimeAlignment_HourlyTimeWindows(t *testing.T) {
+	// Test the time alignment with hourly time windows
+	from, to := computeTimeAlignment(3600, 60)
+	assert.Equal(t, int64(3600), to.Unix()-from.Unix())
+}
+
+func TestTimeAlignment_15minTimeWindows(t *testing.T) {
+	// Test the time alignment with hourly time windows
+	from, to := computeTimeAlignment(3600, 15)
+	assert.Equal(t, int64(900), to.Unix()-from.Unix())
+}
+
+func TestTimeAlignment_15min_HourlyTimeWindows(t *testing.T) {
+	// Test the time alignment with hourly time windows and 15min resolution
+	from, to := computeTimeAlignment(900, 60)
+	assert.Equal(t, int64(3600), to.Unix()-from.Unix())
+}
+
+func TestTimeAlignment_5min_HourlyTimeWindows(t *testing.T) {
+	// Test the time alignment with hourly time windows and 5min resolution
+	from, to := computeTimeAlignment(300, 60)
+	assert.Equal(t, int64(3600), to.Unix()-from.Unix())
+}
+
+func TestTimeAlignment_raw_HourlyTimeWindows(t *testing.T) {
+	// Test the time alignment with hourly time windows and 5min resolution
+	from, to := computeTimeAlignment(-1, 60)
+	assert.Equal(t, int64(3600), to.Unix()-from.Unix())
+}

deployment/cloud-formation-template/deployment.yaml

@@ -42,7 +42,7 @@ Parameters:
 
   Resolution:
       Type: String
-      Description: "Samples resolution data extraction resolution."
+      Description: "Samples resolution data extraction resolution. 'raw' and '1 minute' are not supported for '1 day' scheduling"
       AllowedValues:
         - raw
         - 1 minute

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.1
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.50.1
 	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/oauth2 v0.21.0
 )
 
@@ -29,6 +30,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
 	github.com/aws/smithy-go v1.20.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -62,6 +62,7 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

lambda.go

@@ -81,6 +81,55 @@ func HandleRequest(ctx context.Context, event *AWSS3ImportTrigger) (*string, err
 	}
 
 	// Resolve resolution
+	resolution, err := configureExtractionResolution(logger, paramReader)
+	if err != nil {
+		return nil, err
+	}
+
+	// Resolve scheduling
+	extractionWindowMinutes, err := configureDataExtractionTimeWindow(logger, paramReader)
+	if err != nil {
+		return nil, err
+	}
+
+	if *extractionWindowMinutes > 60 && *resolution <= 60 {
+		logger.Warn("Resolution must be greater than 60 seconds for time windows greater than 60 minutes. Setting resolution to 5 minutes.")
+		defReso := SamplesResolutionSeconds
+		resolution = &defReso
+	}
+
+	logger.Infoln("------ Running import...")
+	if event.Dev || os.Getenv("DEV") == "true" {
+		logger.Infoln("Running in dev mode")
+		os.Setenv("IOT_API_URL", "https://api2.oniudra.cc")
+	}
+	logger.Infoln("key:", *apikey)
+	logger.Infoln("secret:", "*********")
+	if organizationId != "" {
+		logger.Infoln("organizationId:", organizationId)
+	} else {
+		logger.Infoln("organizationId: not set")
+	}
+	if tags != nil {
+		logger.Infoln("tags:", *tags)
+	}
+	if *resolution <= 0 {
+		logger.Infoln("resolution: raw")
+	} else {
+		logger.Infoln("resolution:", *resolution, "seconds")
+	}
+	logger.Infoln("data extraction time windows:", extractionWindowMinutes, "minutes")
+
+	err = importer.StartImport(ctx, logger, *apikey, *apiSecret, organizationId, tags, *resolution, *extractionWindowMinutes, *destinationS3Bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	message := "Data exported successfully"
+	return &message, nil
+}
+
+func configureExtractionResolution(logger *logrus.Entry, paramReader *parameters.ParametersClient) (*int, error) {
 	resolution, err := paramReader.ReadIntConfig(SamplesResoSec)
 	if err != nil {
 		// Possibly this parameter is not set. Try SamplesReso
@@ -108,8 +157,10 @@ func HandleRequest(ctx context.Context, event *AWSS3ImportTrigger) (*string, err
 		logger.Errorf("Resolution %d is invalid", *resolution)
 		return nil, errors.New("resolution must be between -1 and 3600")
 	}
+	return resolution, nil
+}
 
-	// Resolve scheduling
+func configureDataExtractionTimeWindow(logger *logrus.Entry, paramReader *parameters.ParametersClient) (*int, error) {
 	schedule, err := paramReader.ReadConfig(Scheduling)
 	if err != nil {
 		logger.Error("Error reading parameter "+Scheduling, err)
@@ -126,36 +177,7 @@ func HandleRequest(ctx context.Context, event *AWSS3ImportTrigger) (*string, err
 	case "1 day":
 		extractionWindowMinutes = 24 * 60
 	}
-
-	logger.Infoln("------ Running import...")
-	if event.Dev || os.Getenv("DEV") == "true" {
-		logger.Infoln("Running in dev mode")
-		os.Setenv("IOT_API_URL", "https://api2.oniudra.cc")
-	}
-	logger.Infoln("key:", *apikey)
-	logger.Infoln("secret:", "*********")
-	if organizationId != "" {
-		logger.Infoln("organizationId:", organizationId)
-	} else {
-		logger.Infoln("organizationId: not set")
-	}
-	if tags != nil {
-		logger.Infoln("tags:", *tags)
-	}
-	if *resolution <= 0 {
-		logger.Infoln("resolution: raw")
-	} else {
-		logger.Infoln("resolution:", *resolution, "seconds")
-	}
-	logger.Infoln("data extraction time windows:", extractionWindowMinutes, "minutes")
-
-	err = importer.StartImport(ctx, logger, *apikey, *apiSecret, organizationId, tags, *resolution, extractionWindowMinutes, *destinationS3Bucket)
-	if err != nil {
-		return nil, err
-	}
-
-	message := "Data exported successfully"
-	return &message, nil
+	return &extractionWindowMinutes, nil
 }
 
 func main() {