Skip to content

Commit 5644cb0

Browse files
committed
multi stack deployment
1 parent 15ae6ae commit 5644cb0

File tree

4 files changed

+121
-58
lines changed

4 files changed

+121
-58
lines changed

README.md

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ These parameters are filled by CFT at stack creation time and can be adjusted la
6969

7070
| Parameter | Description |
7171
| --------- | ----------- |
72-
| /arduino/s3-importer/iot/api-key | IoT API key |
73-
| /arduino/s3-importer/iot/api-secret | IoT API secret |
74-
| /arduino/s3-importer/iot/org-id | (optional) organization id |
75-
| /arduino/s3-importer/iot/filter/tags | (optional) tags filtering. Syntax: tag=value,tag2=value2 |
76-
| /arduino/s3-importer/iot/samples-resolution | (optional) samples aggregation resolution (1/5/15 minutes, 1 hour, raw) |
77-
| /arduino/s3-importer/destination-bucket | S3 destination bucket |
78-
| /arduino/s3-importer/iot/scheduling | Execution scheduling |
72+
| /arduino/s3-exporter/<stack-name>/iot/api-key | IoT API key |
73+
| /arduino/s3-exporter/<stack-name>/iot/api-secret | IoT API secret |
74+
| /arduino/s3-exporter/<stack-name>/iot/org-id | (optional) organization id |
75+
| /arduino/s3-exporter/<stack-name>/iot/filter/tags | (optional) tags filtering. Syntax: tag=value,tag2=value2 |
76+
| /arduino/s3-exporter/<stack-name>/iot/samples-resolution | (optional) samples aggregation resolution (1/5/15 minutes, 1 hour, raw) |
77+
| /arduino/s3-exporter/<stack-name>/destination-bucket | S3 destination bucket |
78+
| /arduino/s3-exporter/<stack-name>/iot/scheduling | Execution scheduling |
7979

8080
### Tag filtering
8181

@@ -88,7 +88,7 @@ You can use tag filtering if you need to reduce export to a specific set of Thin
8888

8989
![tag 1](docs/tag-1.png)
9090

91-
* Configure tag filter during CFT creation of by editing '/arduino/s3-importer/iot/filter/tags' parameter (syntax: tag1=value1,tag2=value2).
91+
* Configure tag filter during CFT creation of by editing '/arduino/s3-exporter/<stack-name>/iot/filter/tags' parameter (syntax: tag1=value1,tag2=value2).
9292

9393
![tag filter](docs/tag-filter.png)
9494

deployment/cloud-formation-template/deployment.yaml

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,6 @@ AWSTemplateFormatVersion: '2010-09-09'
22
Description: Arduino S3 data exporter. For deployment and architectural details, see https://github.com/arduino/aws-s3-integration
33

44
Parameters:
5-
LambdaFunctionName:
6-
Type: String
7-
Default: 'arduino-s3-csv-data-exporter'
8-
Description: Name of the Lambda function.
9-
105
LambdaCodeS3Bucket:
116
Type: String
127
Description: S3 bucket where the Lambda function ZIP file is stored.
@@ -103,8 +98,7 @@ Resources:
10398
LambdaFunction:
10499
Type: AWS::Lambda::Function
105100
Properties:
106-
FunctionName:
107-
Ref: LambdaFunctionName
101+
FunctionName: !Sub arduino-s3-csv-data-exporter-${AWS::StackName}
108102
Handler: bootstrap
109103
Role: !GetAtt ArduinoS3LambdaExecutionRole.Arn
110104
Code:
@@ -115,12 +109,15 @@ Resources:
115109
Runtime: provided.al2
116110
Timeout: 900
117111
MemorySize: 256
112+
Environment:
113+
Variables:
114+
STACK_NAME: !Sub ${AWS::StackName}
118115

119116
# Parameters in Parameter Store
120117
ApiKeyParameter:
121118
Type: AWS::SSM::Parameter
122119
Properties:
123-
Name: /arduino/s3-importer/iot/api-key
120+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/iot/api-key
124121
Type: String
125122
Value:
126123
Ref: IotApiKey
@@ -129,7 +126,7 @@ Resources:
129126
ApiSecretParameter:
130127
Type: AWS::SSM::Parameter
131128
Properties:
132-
Name: /arduino/s3-importer/iot/api-secret
129+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/iot/api-secret
133130
Type: String
134131
Value:
135132
Ref: IotApiSecret
@@ -138,7 +135,7 @@ Resources:
138135
OrgIdParameter:
139136
Type: AWS::SSM::Parameter
140137
Properties:
141-
Name: /arduino/s3-importer/iot/org-id
138+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/iot/org-id
142139
Type: String
143140
Value:
144141
Ref: IotOrgId
@@ -147,7 +144,7 @@ Resources:
147144
FilterTagsParameter:
148145
Type: AWS::SSM::Parameter
149146
Properties:
150-
Name: /arduino/s3-importer/iot/filter/tags
147+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/iot/filter/tags
151148
Type: String
152149
Value:
153150
Ref: TagFilter
@@ -156,7 +153,7 @@ Resources:
156153
ResolutionParameter:
157154
Type: AWS::SSM::Parameter
158155
Properties:
159-
Name: /arduino/s3-importer/iot/samples-resolution
156+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/iot/samples-resolution
160157
Type: String
161158
Value:
162159
Ref: Resolution
@@ -165,7 +162,7 @@ Resources:
165162
DestinationS3BucketParameter:
166163
Type: AWS::SSM::Parameter
167164
Properties:
168-
Name: /arduino/s3-importer/destination-bucket
165+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/destination-bucket
169166
Type: String
170167
Value:
171168
Ref: DestinationS3Bucket
@@ -174,7 +171,7 @@ Resources:
174171
ExecutionSchedulingParameter:
175172
Type: AWS::SSM::Parameter
176173
Properties:
177-
Name: /arduino/s3-importer/iot/scheduling
174+
Name: !Sub /arduino/s3-exporter/${AWS::StackName}/iot/scheduling
178175
Type: String
179176
Value:
180177
Ref: ExecutionScheduling
@@ -196,8 +193,7 @@ Resources:
196193
LambdaPermissionForEventBridge:
197194
Type: AWS::Lambda::Permission
198195
Properties:
199-
FunctionName:
200-
Ref: LambdaFunctionName
196+
FunctionName: !Sub arduino-s3-csv-data-exporter-${AWS::StackName}
201197
Action: lambda:InvokeFunction
202198
Principal: events.amazonaws.com
203199
SourceArn: !GetAtt EventBridgeRule.Arn

internal/parameters/params.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@ package parameters
1818
import (
1919
"context"
2020
"strconv"
21+
"strings"
2122

2223
"github.com/aws/aws-sdk-go-v2/aws"
2324
"github.com/aws/aws-sdk-go-v2/config"
2425
"github.com/aws/aws-sdk-go-v2/service/ssm"
2526
)
2627

28+
const StackName = "<stack-name>"
29+
2730
type ParametersClient struct {
2831
ssmcl *ssm.Client
2932
}
@@ -47,6 +50,11 @@ func New() (*ParametersClient, error) {
4750
}, nil
4851
}
4952

53+
func (c *ParametersClient) ReadConfigByStack(param, stack string) (*string, error) {
54+
param = strings.ReplaceAll(param, StackName, stack)
55+
return c.ReadConfig(param)
56+
}
57+
5058
func (c *ParametersClient) ReadConfig(param string) (*string, error) {
5159
value, err := c.ssmcl.GetParameter(context.Background(), &ssm.GetParameterInput{
5260
Name: aws.String(param),
@@ -63,6 +71,11 @@ func (c *ParametersClient) ReadConfig(param string) (*string, error) {
6371
return paramValue, nil
6472
}
6573

74+
func (c *ParametersClient) ReadIntConfigByStack(param, stack string) (*int, error) {
75+
param = strings.ReplaceAll(param, StackName, stack)
76+
return c.ReadIntConfig(param)
77+
}
78+
6679
func (c *ParametersClient) ReadIntConfig(param string) (*int, error) {
6780
value, err := c.ssmcl.GetParameter(context.Background(), &ssm.GetParameterInput{
6881
Name: aws.String(param),

lambda.go

Lines changed: 88 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -31,63 +31,105 @@ type AWSS3ImportTrigger struct {
3131
}
3232

3333
const (
34-
ArduinoPrefix = "/arduino/s3-importer"
35-
IoTApiKey = ArduinoPrefix + "/iot/api-key"
36-
IoTApiSecret = ArduinoPrefix + "/iot/api-secret"
37-
IoTApiOrgId = ArduinoPrefix + "/iot/org-id"
38-
IoTApiTags = ArduinoPrefix + "/iot/filter/tags"
39-
SamplesResoSec = ArduinoPrefix + "/iot/samples-resolution-seconds"
40-
SamplesReso = ArduinoPrefix + "/iot/samples-resolution"
41-
Scheduling = ArduinoPrefix + "/iot/scheduling"
42-
DestinationS3Bucket = ArduinoPrefix + "/destination-bucket"
34+
GlobalArduinoPrefix = "/arduino/s3-importer"
35+
36+
// Compatibility parameters for backward compatibility
37+
IoTApiKey = GlobalArduinoPrefix + "/iot/api-key"
38+
IoTApiSecret = GlobalArduinoPrefix + "/iot/api-secret"
39+
IoTApiOrgId = GlobalArduinoPrefix + "/iot/org-id"
40+
IoTApiTags = GlobalArduinoPrefix + "/iot/filter/tags"
41+
SamplesResoSec = GlobalArduinoPrefix + "/iot/samples-resolution-seconds"
42+
SamplesReso = GlobalArduinoPrefix + "/iot/samples-resolution"
43+
Scheduling = GlobalArduinoPrefix + "/iot/scheduling"
44+
DestinationS3Bucket = GlobalArduinoPrefix + "/destination-bucket"
45+
46+
// Per stack parameters
47+
PerStackArduinoPrefix = "/arduino/s3-exporter/" + parameters.StackName
48+
IoTApiKeyStack = PerStackArduinoPrefix + "/iot/api-key"
49+
IoTApiSecretStack = PerStackArduinoPrefix + "/iot/api-secret"
50+
IoTApiOrgIdStack = PerStackArduinoPrefix + "/iot/org-id"
51+
IoTApiTagsStack = PerStackArduinoPrefix + "/iot/filter/tags"
52+
SamplesResoStack = PerStackArduinoPrefix + "/iot/samples-resolution"
53+
SchedulingStack = PerStackArduinoPrefix + "/iot/scheduling"
54+
DestinationS3BucketStack = PerStackArduinoPrefix + "/destination-bucket"
55+
4356
SamplesResolutionSeconds = 300
4457
DefaultTimeExtractionWindowMinutes = 60
4558
)
4659

4760
func HandleRequest(ctx context.Context, event *AWSS3ImportTrigger) (*string, error) {
4861

4962
logger := logrus.NewEntry(logrus.New())
63+
stackName := os.Getenv("STACK_NAME")
5064

65+
var apikey *string
66+
var apiSecret *string
67+
var destinationS3Bucket *string
5168
var tags *string
69+
var orgId *string
70+
var err error
5271

5372
logger.Infoln("------ Reading parameters from SSM")
5473
paramReader, err := parameters.New()
5574
if err != nil {
5675
return nil, err
5776
}
58-
apikey, err := paramReader.ReadConfig(IoTApiKey)
59-
if err != nil {
60-
logger.Error("Error reading parameter "+IoTApiKey, err)
61-
}
62-
apiSecret, err := paramReader.ReadConfig(IoTApiSecret)
63-
if err != nil {
64-
logger.Error("Error reading parameter "+IoTApiSecret, err)
65-
}
66-
destinationS3Bucket, err := paramReader.ReadConfig(DestinationS3Bucket)
67-
if err != nil || destinationS3Bucket == nil || *destinationS3Bucket == "" {
68-
logger.Error("Error reading parameter "+DestinationS3Bucket, err)
77+
78+
if stackName != "" {
79+
logger.Infoln("------ Configured stack: " + stackName)
80+
apikey, err = paramReader.ReadConfigByStack(IoTApiKeyStack, stackName)
81+
if err != nil {
82+
logger.Error("Error reading parameter "+IoTApiKeyStack, err)
83+
}
84+
apiSecret, err = paramReader.ReadConfigByStack(IoTApiSecretStack, stackName)
85+
if err != nil {
86+
logger.Error("Error reading parameter "+IoTApiSecretStack, err)
87+
}
88+
destinationS3Bucket, err = paramReader.ReadConfigByStack(DestinationS3BucketStack, stackName)
89+
if err != nil || destinationS3Bucket == nil || *destinationS3Bucket == "" {
90+
logger.Error("Error reading parameter "+DestinationS3BucketStack, err)
91+
}
92+
orgId, _ = paramReader.ReadConfigByStack(IoTApiOrgIdStack, stackName)
93+
tagsParam, _ := paramReader.ReadConfigByStack(IoTApiTagsStack, stackName)
94+
if tagsParam != nil {
95+
tags = tagsParam
96+
}
97+
} else {
98+
apikey, err = paramReader.ReadConfig(IoTApiKey)
99+
if err != nil {
100+
logger.Error("Error reading parameter "+IoTApiKey, err)
101+
}
102+
apiSecret, err = paramReader.ReadConfig(IoTApiSecret)
103+
if err != nil {
104+
logger.Error("Error reading parameter "+IoTApiSecret, err)
105+
}
106+
destinationS3Bucket, err = paramReader.ReadConfig(DestinationS3Bucket)
107+
if err != nil || destinationS3Bucket == nil || *destinationS3Bucket == "" {
108+
logger.Error("Error reading parameter "+DestinationS3Bucket, err)
109+
}
110+
orgId, _ = paramReader.ReadConfig(IoTApiOrgId)
111+
tagsParam, _ := paramReader.ReadConfig(IoTApiTags)
112+
if tagsParam != nil {
113+
tags = tagsParam
114+
}
69115
}
70-
origId, _ := paramReader.ReadConfig(IoTApiOrgId)
116+
71117
organizationId := ""
72-
if origId != nil {
73-
organizationId = *origId
118+
if orgId != nil {
119+
organizationId = *orgId
74120
}
75121
if apikey == nil || apiSecret == nil {
76122
return nil, errors.New("key and secret are required")
77123
}
78-
tagsParam, _ := paramReader.ReadConfig(IoTApiTags)
79-
if tagsParam != nil {
80-
tags = tagsParam
81-
}
82124

83125
// Resolve resolution
84-
resolution, err := configureExtractionResolution(logger, paramReader)
126+
resolution, err := configureExtractionResolution(logger, paramReader, stackName)
85127
if err != nil {
86128
return nil, err
87129
}
88130

89131
// Resolve scheduling
90-
extractionWindowMinutes, err := configureDataExtractionTimeWindow(logger, paramReader)
132+
extractionWindowMinutes, err := configureDataExtractionTimeWindow(logger, paramReader, stackName)
91133
if err != nil {
92134
return nil, err
93135
}
@@ -98,7 +140,7 @@ func HandleRequest(ctx context.Context, event *AWSS3ImportTrigger) (*string, err
98140
resolution = &defReso
99141
}
100142

101-
logger.Infoln("------ Running import...")
143+
logger.Infoln("------ Running import")
102144
if event.Dev || os.Getenv("DEV") == "true" {
103145
logger.Infoln("Running in dev mode")
104146
os.Setenv("IOT_API_URL", "https://api2.oniudra.cc")
@@ -129,8 +171,14 @@ func HandleRequest(ctx context.Context, event *AWSS3ImportTrigger) (*string, err
129171
return &message, nil
130172
}
131173

132-
func configureExtractionResolution(logger *logrus.Entry, paramReader *parameters.ParametersClient) (*int, error) {
133-
resolution, err := paramReader.ReadIntConfig(SamplesResoSec)
174+
func configureExtractionResolution(logger *logrus.Entry, paramReader *parameters.ParametersClient, stack string) (*int, error) {
175+
var resolution *int
176+
var err error
177+
if stack != "" {
178+
resolution, err = paramReader.ReadIntConfigByStack(SamplesResoStack, stack)
179+
} else {
180+
resolution, err = paramReader.ReadIntConfig(SamplesReso)
181+
}
134182
if err != nil {
135183
// Possibly this parameter is not set. Try SamplesReso
136184
res, err := paramReader.ReadConfig(SamplesReso)
@@ -160,8 +208,14 @@ func configureExtractionResolution(logger *logrus.Entry, paramReader *parameters
160208
return resolution, nil
161209
}
162210

163-
func configureDataExtractionTimeWindow(logger *logrus.Entry, paramReader *parameters.ParametersClient) (*int, error) {
164-
schedule, err := paramReader.ReadConfig(Scheduling)
211+
func configureDataExtractionTimeWindow(logger *logrus.Entry, paramReader *parameters.ParametersClient, stack string) (*int, error) {
212+
var schedule *string
213+
var err error
214+
if stack != "" {
215+
schedule, err = paramReader.ReadConfigByStack(SchedulingStack, stack)
216+
} else {
217+
schedule, err = paramReader.ReadConfig(Scheduling)
218+
}
165219
if err != nil {
166220
logger.Error("Error reading parameter "+Scheduling, err)
167221
return nil, err

0 commit comments

Comments
 (0)