Skip to content

Commit d2d568b

Browse files
medmundsmedmunds
committed
SendGrid: simplify personalizations processing; stop using "sections"
* Rework and simplify personalizations code (that had grown convoluted through several feature additions). * Stop putting merge_global_data in legacy template "sections"; instead just merge it into individual personalization substitutions like we do for dynamic templates. (The "sections" version didn't add any functionality, had the potential for conflicts with the user's own template section tags, and was needlessly complex.)
1 parent f89d92b commit d2d568b

File tree

3 files changed

+70
-119
lines changed

3 files changed

+70
-119
lines changed

‎CHANGELOG.rst

+3
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ Fixes
6060
(but no To addresses). Also, `message.anymail_status.recipients[email]` now includes
6161
send status for Cc and Bcc recipients. (Thanks to `@ailionx`_ for reporting the error.)
6262

63+
* **SendGrid:** With legacy templates, stop (ab)using "sections" for merge_global_data.
64+
This avoids potential conflicts with a template's own use of SendGrid section tags.
65+
6366

6467
v5.0
6568
----

‎anymail/backends/sendgrid.py

+55-99
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,9 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
7575
self.use_dynamic_template = False # how to represent merge_data
7676
self.message_id = None # Message-ID -- assigned in serialize_data unless provided in headers
7777
self.merge_field_format = backend.merge_field_format
78-
self.merge_data = None # late-bound per-recipient data
79-
self.merge_global_data = None
80-
self.merge_metadata = None
78+
self.merge_data = {} # late-bound per-recipient data
79+
self.merge_global_data = {}
80+
self.merge_metadata = {}
8181

8282
http_headers = kwargs.pop('headers', {})
8383
http_headers['Authorization'] = 'Bearer %s' % backend.api_key
@@ -101,6 +101,8 @@ def serialize_data(self):
101101

102102
if self.generate_message_id:
103103
self.set_anymail_id()
104+
if self.is_batch():
105+
self.expand_personalizations_for_batch()
104106
self.build_merge_data()
105107
self.build_merge_metadata()
106108

@@ -115,118 +117,72 @@ def set_anymail_id(self):
115117
self.message_id = str(uuid.uuid4())
116118
self.data.setdefault("custom_args", {})["anymail_id"] = self.message_id
117119

118-
def build_merge_data(self):
119-
if self.use_dynamic_template:
120-
self.build_merge_data_dynamic()
121-
else:
122-
self.build_merge_data_legacy()
123-
124-
def build_merge_data_dynamic(self):
125-
"""Set personalizations[...]['dynamic_template_data']"""
126-
if self.merge_global_data is not None:
127-
assert len(self.data["personalizations"]) == 1
128-
self.data["personalizations"][0].setdefault(
129-
"dynamic_template_data", {}).update(self.merge_global_data)
120+
def expand_personalizations_for_batch(self):
121+
"""Split data["personalizations"] into individual message for each recipient"""
122+
assert len(self.data["personalizations"]) == 1
123+
base_personalization = self.data["personalizations"].pop()
124+
to_list = base_personalization.pop("to") # {email, name?} for each message.to
125+
for recipient in to_list:
126+
personalization = base_personalization.copy()
127+
personalization["to"] = [recipient]
128+
self.data["personalizations"].append(personalization)
130129

131-
if self.merge_data is not None:
132-
# Burst apart each to-email in personalizations[0] into a separate
133-
# personalization, and add merge_data for that recipient
134-
assert len(self.data["personalizations"]) == 1
135-
base_personalizations = self.data["personalizations"].pop()
136-
to_list = base_personalizations.pop("to") # {email, name?} for each message.to
137-
for recipient in to_list:
138-
personalization = base_personalizations.copy() # captures cc, bcc, merge_global_data, esp_extra
139-
personalization["to"] = [recipient]
140-
try:
141-
recipient_data = self.merge_data[recipient["email"]]
142-
except KeyError:
143-
pass # no merge_data for this recipient
144-
else:
145-
if "dynamic_template_data" in personalization:
146-
# merge per-recipient data into (copy of) merge_global_data
147-
personalization["dynamic_template_data"] = personalization["dynamic_template_data"].copy()
148-
personalization["dynamic_template_data"].update(recipient_data)
149-
else:
150-
personalization["dynamic_template_data"] = recipient_data
151-
self.data["personalizations"].append(personalization)
152-
153-
def build_merge_data_legacy(self):
154-
"""Set personalizations[...]['substitutions'] and data['sections']"""
130+
def build_merge_data(self):
131+
if self.merge_data or self.merge_global_data:
132+
# Always build dynamic_template_data first,
133+
# then convert it to legacy template format if needed
134+
for personalization in self.data["personalizations"]:
135+
assert len(personalization["to"]) == 1
136+
recipient_email = personalization["to"][0]["email"]
137+
dynamic_template_data = self.merge_global_data.copy()
138+
dynamic_template_data.update(self.merge_data.get(recipient_email, {}))
139+
if dynamic_template_data:
140+
personalization["dynamic_template_data"] = dynamic_template_data
141+
142+
if not self.use_dynamic_template:
143+
self.convert_dynamic_template_data_to_legacy_substitutions()
144+
145+
def convert_dynamic_template_data_to_legacy_substitutions(self):
146+
"""Change personalizations[...]['dynamic_template_data'] to ...['substitutions]"""
155147
merge_field_format = self.merge_field_format or '{}'
156148

157-
if self.merge_data is not None:
158-
# Burst apart each to-email in personalizations[0] into a separate
159-
# personalization, and add merge_data for that recipient
160-
assert len(self.data["personalizations"]) == 1
161-
base_personalizations = self.data["personalizations"].pop()
162-
to_list = base_personalizations.pop("to") # {email, name?} for each message.to
163-
all_fields = set()
164-
for recipient in to_list:
165-
personalization = base_personalizations.copy() # captures cc, bcc, and any esp_extra
166-
personalization["to"] = [recipient]
167-
try:
168-
recipient_data = self.merge_data[recipient["email"]]
169-
personalization["substitutions"] = {merge_field_format.format(field): data
170-
for field, data in recipient_data.items()}
171-
all_fields = all_fields.union(recipient_data.keys())
172-
except KeyError:
173-
pass # no merge_data for this recipient
174-
self.data["personalizations"].append(personalization)
175-
176-
if self.merge_field_format is None and len(all_fields) and all(field.isalnum() for field in all_fields):
149+
all_merge_fields = set()
150+
for personalization in self.data["personalizations"]:
151+
try:
152+
dynamic_template_data = personalization.pop("dynamic_template_data")
153+
except KeyError:
154+
pass # no substitutions for this recipient
155+
else:
156+
# Convert dynamic_template_data keys for substitutions, using merge_field_format
157+
personalization["substitutions"] = {
158+
merge_field_format.format(field): data
159+
for field, data in dynamic_template_data.items()}
160+
all_merge_fields.update(dynamic_template_data.keys())
161+
162+
if self.merge_field_format is None:
163+
if all_merge_fields and all(field.isalnum() for field in all_merge_fields):
177164
warnings.warn(
178165
"Your SendGrid merge fields don't seem to have delimiters, "
179166
"which can cause unexpected results with Anymail's merge_data. "
180167
"Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
181168
AnymailWarning)
182169

183-
if self.merge_global_data is not None:
184-
# (merge into any existing 'sections' from esp_extra)
185-
self.data.setdefault("sections", {}).update({
186-
merge_field_format.format(field): data
187-
for field, data in self.merge_global_data.items()
188-
})
189-
190-
# Confusingly, "Section tags have to be contained within a Substitution tag"
191-
# (https://sendgrid.com/docs/API_Reference/SMTP_API/section_tags.html),
192-
# so we need to insert a "-field-": "-field-" identity fallback for each
193-
# missing global field in the recipient substitutions...
194-
global_fields = [merge_field_format.format(field)
195-
for field in self.merge_global_data.keys()]
196-
for personalization in self.data["personalizations"]:
197-
substitutions = personalization.setdefault("substitutions", {})
198-
substitutions.update({field: field for field in global_fields
199-
if field not in substitutions})
200-
201-
if (self.merge_field_format is None and
202-
all(field.isalnum() for field in self.merge_global_data.keys())):
170+
if self.merge_global_data and all(field.isalnum() for field in self.merge_global_data.keys()):
203171
warnings.warn(
204172
"Your SendGrid global merge fields don't seem to have delimiters, "
205173
"which can cause unexpected results with Anymail's merge_data. "
206174
"Search SENDGRID_MERGE_FIELD_FORMAT in the Anymail docs for more info.",
207175
AnymailWarning)
208176

209177
def build_merge_metadata(self):
210-
if self.merge_metadata is None:
211-
return
212-
213-
if self.merge_data is None:
214-
# Burst apart each to-email in personalizations[0] into a separate
215-
# personalization, and add merge_metadata for that recipient
216-
assert len(self.data["personalizations"]) == 1
217-
base_personalizations = self.data["personalizations"].pop()
218-
to_list = base_personalizations.pop("to") # {email, name?} for each message.to
219-
for recipient in to_list:
220-
personalization = base_personalizations.copy() # captures cc, bcc, and any esp_extra
221-
personalization["to"] = [recipient]
222-
self.data["personalizations"].append(personalization)
223-
224-
for personalization in self.data["personalizations"]:
225-
recipient_email = personalization["to"][0]["email"]
226-
recipient_metadata = self.merge_metadata.get(recipient_email)
227-
if recipient_metadata:
228-
recipient_custom_args = self.transform_metadata(recipient_metadata)
229-
personalization["custom_args"] = recipient_custom_args
178+
if self.merge_metadata:
179+
for personalization in self.data["personalizations"]:
180+
assert len(personalization["to"]) == 1
181+
recipient_email = personalization["to"][0]["email"]
182+
recipient_metadata = self.merge_metadata.get(recipient_email)
183+
if recipient_metadata:
184+
recipient_custom_args = self.transform_metadata(recipient_metadata)
185+
personalization["custom_args"] = recipient_custom_args
230186

231187
#
232188
# Payload construction

‎tests/test_sendgrid_backend.py

+12-20
Original file line numberDiff line numberDiff line change
@@ -515,18 +515,15 @@ def test_legacy_merge_data(self):
515515
{'to': [{'email': '[email protected]'}],
516516
'cc': [{'email': '[email protected]'}], # all recipients get the cc
517517
'substitutions': {':name': "Alice", ':group': "Developers",
518-
':site': ":site"}}, # tell SG to look for global field in 'sections'
518+
':site': "ExampleCo"}}, # merge_global_data merged
519519
{'to': [{'email': '[email protected]', 'name': '"Bob"'}],
520520
'cc': [{'email': '[email protected]'}],
521-
'substitutions': {':name': "Bob", ':group': ":group", ':site': ":site"}},
521+
'substitutions': {':name': "Bob", ':group': "Users", ':site': "ExampleCo"}},
522522
{'to': [{'email': '[email protected]'}],
523523
'cc': [{'email': '[email protected]'}],
524-
'substitutions': {':group': ":group", ':site': ":site"}}, # look for global fields in 'sections'
524+
'substitutions': {':group': "Users", ':site': "ExampleCo"}},
525525
])
526-
self.assertEqual(data['sections'], {
527-
':group': "Users",
528-
':site': "ExampleCo",
529-
})
526+
self.assertNotIn('sections', data) # 'sections' no longer used for merge_global_data
530527

531528
@override_settings(ANYMAIL_SENDGRID_MERGE_FIELD_FORMAT=":{}") # :field as shown in SG examples
532529
def test_legacy_merge_field_format_setting(self):
@@ -541,11 +538,11 @@ def test_legacy_merge_field_format_setting(self):
541538
data = self.get_api_call_json()
542539
self.assertEqual(data['personalizations'], [
543540
{'to': [{'email': '[email protected]'}],
544-
'substitutions': {':name': "Alice", ':group': "Developers", ':site': ":site"}}, # keys changed to :field
541+
'substitutions': {':name': "Alice", ':group': "Developers", # keys changed to :field
542+
':site': "ExampleCo"}},
545543
{'to': [{'email': '[email protected]', 'name': '"Bob"'}],
546-
'substitutions': {':name': "Bob", ':site': ":site"}}
544+
'substitutions': {':name': "Bob", ':site': "ExampleCo"}}
547545
])
548-
self.assertEqual(data['sections'], {':site': "ExampleCo"})
549546

550547
def test_legacy_merge_field_format_esp_extra(self):
551548
# Provide merge field delimiters for an individual message
@@ -560,11 +557,10 @@ def test_legacy_merge_field_format_esp_extra(self):
560557
data = self.get_api_call_json()
561558
self.assertEqual(data['personalizations'], [
562559
{'to': [{'email': '[email protected]'}],
563-
'substitutions': {'*|name|*': "Alice", '*|group|*': "Developers", '*|site|*': "*|site|*"}},
560+
'substitutions': {'*|name|*': "Alice", '*|group|*': "Developers", '*|site|*': "ExampleCo"}},
564561
{'to': [{'email': '[email protected]', 'name': '"Bob"'}],
565-
'substitutions': {'*|name|*': "Bob", '*|site|*': "*|site|*"}}
562+
'substitutions': {'*|name|*': "Bob", '*|site|*': "ExampleCo"}}
566563
])
567-
self.assertEqual(data['sections'], {'*|site|*': "ExampleCo"})
568564
# Make sure our esp_extra merge_field_format doesn't get sent to SendGrid API:
569565
self.assertNotIn('merge_field_format', data)
570566

@@ -682,20 +678,16 @@ def test_merge_metadata_with_legacy_template(self):
682678
{'to': [{'email': '[email protected]'}],
683679
'cc': [{'email': '[email protected]'}], # all recipients get the cc
684680
'custom_args': {'order_id': '123'},
685-
'substitutions': {':name': "Alice", ':group': "Developers", ':site': ":site"}},
681+
'substitutions': {':name': "Alice", ':group': "Developers", ':site': "ExampleCo"}},
686682
{'to': [{'email': '[email protected]', 'name': '"Bob"'}],
687683
'cc': [{'email': '[email protected]'}],
688684
'custom_args': {'order_id': '678', 'tier': 'premium'},
689-
'substitutions': {':name': "Bob", ':group': ":group", ':site': ":site"}},
685+
'substitutions': {':name': "Bob", ':group': "Users", ':site': "ExampleCo"}},
690686
{'to': [{'email': '[email protected]'}],
691687
'cc': [{'email': '[email protected]'}],
692688
# no custom_args
693-
'substitutions': {':group': ":group", ':site': ":site"}},
689+
'substitutions': {':group': "Users", ':site': "ExampleCo"}},
694690
])
695-
self.assertEqual(data['sections'], {
696-
':group': "Users",
697-
':site': "ExampleCo",
698-
})
699691

700692
@override_settings(ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID=False) # else we force custom_args
701693
def test_default_omits_options(self):

0 commit comments

Comments
 (0)