SendGrid: change message_id from Message-ID/smtp-id to UUID anymail_id

SendGrid does not always correctly provide the sent Message-ID header value 
to a tracking webhook's smtp-id field, making it unreliable to use for Anymail's 
`message_id`.

Instead, generate a UUID `message_id` for Anymail tracking, and pass it from 
send to webhooks in SendGrid custom args as anymail_id.

Webhooks will fall back to smtp-id for compatibility with previously-sent 
messages that didn't have an anymail_id custom arg.

Fixes #108

Author: joshkersey

anymail/backends/sendgrid.py

@@ -1,7 +1,7 @@
+import uuid
 from email.utils import quote as rfc822_quote
 import warnings
 
-from django.core.mail import make_msgid
 from requests.structures import CaseInsensitiveDict
 
 from .base_requests import AnymailRequestsBackend, RequestsPayload
@@ -99,36 +99,19 @@ def serialize_data(self):
         """Performs any necessary serialization on self.data, and returns the result."""
 
         if self.generate_message_id:
-            self.ensure_message_id()
+            self.set_anymail_id()
         self.build_merge_data()
 
         if not self.data["headers"]:
             del self.data["headers"]  # don't send empty headers
 
         return self.serialize_json(self.data)
 
-    def ensure_message_id(self):
-        """Ensure message has a known Message-ID for later event tracking"""
-        if "Message-ID" not in self.data["headers"]:
-            # Only make our own if caller hasn't already provided one
-            self.data["headers"]["Message-ID"] = self.make_message_id()
-        self.message_id = self.data["headers"]["Message-ID"]
-
-        # Workaround for missing message ID (smtp-id) in SendGrid engagement events
-        # (click and open tracking): because unique_args get merged into the raw event
-        # record, we can supply the 'smtp-id' field for any events missing it.
-        self.data.setdefault("custom_args", {})["smtp-id"] = self.message_id
-
-    def make_message_id(self):
-        """Returns a Message-ID that could be used for this payload
-
-        Tries to use the from_email's domain as the Message-ID's domain
-        """
-        try:
-            _, domain = self.data["from"]["email"].split("@")
-        except (AttributeError, KeyError, TypeError, ValueError):
-            domain = None
-        return make_msgid(domain=domain)
+    def set_anymail_id(self):
+        """Ensure message has a known anymail_id for later event tracking"""
+
+        self.message_id = str(uuid.uuid4())
+        self.data.setdefault("custom_args", {})["anymail_id"] = self.message_id
 
     def build_merge_data(self):
         """Set personalizations[...]['substitutions'] and data['sections']"""

anymail/backends/sendgrid_v2.py

@@ -1,6 +1,6 @@
+import uuid
 import warnings
 
-from django.core.mail import make_msgid
 from requests.structures import CaseInsensitiveDict
 
 from ..exceptions import AnymailConfigurationError, AnymailRequestsAPIError, AnymailWarning
@@ -99,7 +99,7 @@ def serialize_data(self):
         """Performs any necessary serialization on self.data, and returns the result."""
 
         if self.generate_message_id:
-            self.ensure_message_id()
+            self.set_anymail_id()
 
         self.build_merge_data()
         if self.merge_data is not None:
@@ -136,29 +136,11 @@ def serialize_data(self):
 
         return self.data
 
-    def ensure_message_id(self):
-        """Ensure message has a known Message-ID for later event tracking"""
-        headers = self.data["headers"]
-        if "Message-ID" not in headers:
-            # Only make our own if caller hasn't already provided one
-            headers["Message-ID"] = self.make_message_id()
-        self.message_id = headers["Message-ID"]
+    def set_anymail_id(self):
+        """Ensure message has a known anymail_id for later event tracking"""
 
-        # Workaround for missing message ID (smtp-id) in SendGrid engagement events
-        # (click and open tracking): because unique_args get merged into the raw event
-        # record, we can supply the 'smtp-id' field for any events missing it.
-        self.smtpapi.setdefault('unique_args', {})['smtp-id'] = self.message_id
-
-    def make_message_id(self):
-        """Returns a Message-ID that could be used for this payload
-
-        Tries to use the from_email's domain as the Message-ID's domain
-        """
-        try:
-            _, domain = self.data["from"].split("@")
-        except (AttributeError, KeyError, TypeError, ValueError):
-            domain = None
-        return make_msgid(domain=domain)
+        self.message_id = str(uuid.uuid4())
+        self.smtpapi.setdefault('unique_args', {})["anymail_id"] = self.message_id
 
     def build_merge_data(self):
         """Set smtpapi['sub'] and ['section']"""

anymail/webhooks/sendgrid.py

@@ -72,7 +72,7 @@ def esp_to_anymail_event(self, esp_event):
         return AnymailTrackingEvent(
             event_type=event_type,
             timestamp=timestamp,
-            message_id=esp_event.get('smtp-id', None),
+            message_id=esp_event.get('anymail_id', esp_event.get('smtp-id')),  # backwards compatibility
             event_id=esp_event.get('sg_event_id', None),
             recipient=esp_event.get('email', None),
             reject_reason=reject_reason,
@@ -86,6 +86,7 @@ def esp_to_anymail_event(self, esp_event):
 
     # Known keys in SendGrid events (used to recover metadata above)
     sendgrid_event_keys = {
+        'anymail_id',
         'asm_group_id',
         'attempt',  # MTA deferred count
         'category',

tests/test_sendgrid_backend.py

@@ -55,10 +55,8 @@ def test_send_mail(self):
         self.assertEqual(data['personalizations'], [{
             'to': [{'email': "[email protected]"}],
         }])
-        # make sure backend assigned a Message-ID for event tracking
-        self.assertRegex(data['headers']['Message-ID'], r'\<.+@sender\.example\.com\>')  # id uses from_email's domain
-        # make sure we added the Message-ID to custom_args for event notification
-        self.assertEqual(data['headers']['Message-ID'], data['custom_args']['smtp-id'])
+        # make sure the backend assigned the anymail_id for event tracking and notification
+        self.assertUUIDIsValid(data['custom_args']['anymail_id'])
 
     def test_name_addr(self):
         """Make sure RFC2822 name-addr format (with display-name) is allowed
@@ -119,9 +117,7 @@ def test_email_message(self):
             'Message-ID': "<[email protected]>",
         })
         # make sure custom Message-ID also added to custom_args
-        self.assertEqual(data['custom_args'], {
-            'smtp-id': "<[email protected]>",
-        })
+        self.assertUUIDIsValid(data['custom_args']['anymail_id'])
 
     def test_html_message(self):
         text_content = 'This is an important message.'
@@ -345,7 +341,7 @@ def test_metadata(self):
         self.message.metadata = {'user_id': "12345", 'items': 6, 'float': 98.6, 'long': longtype(123)}
         self.message.send()
         data = self.get_api_call_json()
-        data['custom_args'].pop('smtp-id', None)  # remove Message-ID we added as tracking workaround
+        data['custom_args'].pop('anymail_id', None)  # remove Message-ID we added as tracking workaround
         self.assertEqual(data['custom_args'], {'user_id': "12345",
                                                'items': "6",  # int converted to a string,
                                                'float': "98.6",  # float converted to a string (watch binary rounding!)
@@ -579,7 +575,7 @@ def test_send_attaches_anymail_status(self):
         sent = msg.send()
         self.assertEqual(sent, 1)
         self.assertEqual(msg.anymail_status.status, {'queued'})
-        self.assertRegex(msg.anymail_status.message_id, r'\<.+@example\.com\>')  # don't know exactly what it'll be
+        self.assertUUIDIsValid(msg.anymail_status.message_id)  # don't know exactly what it'll be
         self.assertEqual(msg.anymail_status.recipients['[email protected]'].status, 'queued')
         self.assertEqual(msg.anymail_status.recipients['[email protected]'].message_id,
                          msg.anymail_status.message_id)

tests/test_sendgrid_v2_backend.py

@@ -63,12 +63,9 @@ def test_send_mail(self):
         self.assertEqual(data['text'], "Here is the message.")
         self.assertEqual(data['from'], "[email protected]")
         self.assertEqual(data['to'], ["[email protected]"])
-        # make sure backend assigned a Message-ID for event tracking
-        email_headers = json.loads(data['headers'])
-        self.assertRegex(email_headers['Message-ID'], r'\<.+@sender\.example\.com\>')  # id uses from_email's domain
-        # make sure we added the Message-ID to unique_args for event notification
+        # make sure the backend assigned the anymail_id to unique_args for event tracking and notification
         smtpapi = self.get_smtpapi()
-        self.assertEqual(email_headers['Message-ID'], smtpapi['unique_args']['smtp-id'])
+        self.assertUUIDIsValid(smtpapi['unique_args']['anymail_id'])
 
     @override_settings(ANYMAIL={'SENDGRID_USERNAME': 'sg_username', 'SENDGRID_PASSWORD': 'sg_password'})
     def test_user_pass_auth(self):
@@ -129,10 +126,9 @@ def test_email_message(self):
             'Reply-To': '[email protected]',
             'X-MyHeader': 'my value',
         })
-        # make sure custom Message-ID also added to unique_args
-        self.assertJSONEqual(data['x-smtpapi'], {
-            'unique_args': {'smtp-id': '<[email protected]>'}
-        })
+        # make sure anymail_id also added to unique_args
+        smtpapi_json = json.loads(data['x-smtpapi'])
+        self.assertUUIDIsValid(smtpapi_json['unique_args']['anymail_id'])
 
     def test_html_message(self):
         text_content = 'This is an important message.'
@@ -293,8 +289,7 @@ def test_suppress_empty_address_lists(self):
         self.assertNotIn('ccname', data)
         self.assertNotIn('bcc', data)
         self.assertNotIn('bccname', data)
-        headers = json.loads(data['headers'])
-        self.assertNotIn('Reply-To', headers)
+        self.assertNotIn('headers', data)
 
         # Test empty `to` -- but send requires at least one recipient somewhere (like cc)
         self.message.to = []
@@ -354,7 +349,7 @@ def test_metadata(self):
         self.message.metadata = {'user_id': "12345", 'items': 6}
         self.message.send()
         smtpapi = self.get_smtpapi()
-        smtpapi['unique_args'].pop('smtp-id', None)  # remove Message-ID we added as tracking workaround
+        smtpapi['unique_args'].pop('anymail_id', None)  # remove Message-ID we added as tracking workaround
         self.assertEqual(smtpapi['unique_args'], {'user_id': "12345", 'items': 6})
 
     def test_send_at(self):
@@ -565,7 +560,7 @@ def test_send_attaches_anymail_status(self):
         sent = msg.send()
         self.assertEqual(sent, 1)
         self.assertEqual(msg.anymail_status.status, {'queued'})
-        self.assertRegex(msg.anymail_status.message_id, r'\<.+@example\.com\>')  # don't know exactly what it'll be
+        self.assertUUIDIsValid(msg.anymail_status.message_id)
         self.assertEqual(msg.anymail_status.recipients['[email protected]'].status, 'queued')
         self.assertEqual(msg.anymail_status.recipients['[email protected]'].message_id,
                          msg.anymail_status.message_id)

tests/test_sendgrid_webhooks.py

@@ -23,7 +23,7 @@ def test_processed_event(self):
         raw_events = [{
             "email": "[email protected]",
             "timestamp": 1461095246,
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "3c2f4df8-c6dd-4cd2-9b91-6582b81a0349",
             "sg_event_id": "ZyjAM5rnQmuI1KFInHQ3Nw",
             "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
             "event": "processed",
@@ -41,7 +41,7 @@ def test_processed_event(self):
         self.assertEqual(event.event_type, "queued")
         self.assertEqual(event.timestamp, datetime(2016, 4, 19, 19, 47, 26, tzinfo=utc))
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "3c2f4df8-c6dd-4cd2-9b91-6582b81a0349")
         self.assertEqual(event.event_id, "ZyjAM5rnQmuI1KFInHQ3Nw")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.tags, ["tag1", "tag2"])
@@ -57,7 +57,7 @@ def test_delivered_event(self):
             "event": "delivered",
             "email": "[email protected]",
             "timestamp": 1461095250,
-            "smtp-id": "<[email protected]>"
+            "anymail_id": "4ab185c2-0171-492f-9ce0-27de258efc99"
         }]
         response = self.client.post('/anymail/sendgrid/tracking/',
                                     content_type='application/json', data=json.dumps(raw_events))
@@ -69,7 +69,7 @@ def test_delivered_event(self):
         self.assertEqual(event.event_type, "delivered")
         self.assertEqual(event.timestamp, datetime(2016, 4, 19, 19, 47, 30, tzinfo=utc))
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "4ab185c2-0171-492f-9ce0-27de258efc99")
         self.assertEqual(event.event_id, "nOSv8m0eTQ-vxvwNwt3fZQ")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.mta_response, "250 2.0.0 OK 1461095248 m143si2210036ioe.159 - gsmtp ")
@@ -79,7 +79,7 @@ def test_delivered_event(self):
     def test_dropped_invalid_event(self):
         raw_events = [{
             "email": "invalid@invalid",
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "c74002d9-7ccb-4f67-8b8c-766cec03c9a6",
             "timestamp": 1461095250,
             "sg_event_id": "3NPOePGOTkeM_U3fgWApfg",
             "sg_message_id": "filter0093p1las1.9128.5717FB8127.0",
@@ -95,7 +95,7 @@ def test_dropped_invalid_event(self):
         self.assertIsInstance(event, AnymailTrackingEvent)
         self.assertEqual(event.event_type, "rejected")
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "c74002d9-7ccb-4f67-8b8c-766cec03c9a6")
         self.assertEqual(event.event_id, "3NPOePGOTkeM_U3fgWApfg")
         self.assertEqual(event.recipient, "invalid@invalid")
         self.assertEqual(event.reject_reason, "invalid")
@@ -104,7 +104,7 @@ def test_dropped_invalid_event(self):
     def test_dropped_unsubscribed_event(self):
         raw_events = [{
             "email": "[email protected]",
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "a36ec0f9-aabe-45c7-9a84-3e17afb5cb65",
             "timestamp": 1461095250,
             "sg_event_id": "oxy9OLwMTAy5EsuZn1qhIg",
             "sg_message_id": "filter0199p1las1.4745.5717FB6F5.0",
@@ -120,7 +120,7 @@ def test_dropped_unsubscribed_event(self):
         self.assertIsInstance(event, AnymailTrackingEvent)
         self.assertEqual(event.event_type, "rejected")
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "a36ec0f9-aabe-45c7-9a84-3e17afb5cb65")
         self.assertEqual(event.event_id, "oxy9OLwMTAy5EsuZn1qhIg")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.reject_reason, "unsubscribed")
@@ -137,7 +137,7 @@ def test_bounce_event(self):
             "event": "bounce",
             "email": "[email protected]",
             "timestamp": 1461095250,
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "de212213-bb66-4302-8f3f-20acdb7a104e",
             "type": "bounce"
         }]
         response = self.client.post('/anymail/sendgrid/tracking/',
@@ -149,7 +149,7 @@ def test_bounce_event(self):
         self.assertIsInstance(event, AnymailTrackingEvent)
         self.assertEqual(event.event_type, "bounced")
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "de212213-bb66-4302-8f3f-20acdb7a104e")
         self.assertEqual(event.event_id, "lC0Rc-FuQmKbnxCWxX1jRQ")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.mta_response, "550 5.1.1 The email account that you tried to reach does not exist.")
@@ -163,7 +163,7 @@ def test_deferred_event(self):
             "email": "[email protected]",
             "attempt": "1",
             "timestamp": 1461200990,
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "ccf83222-0d7e-4542-8beb-893122afa757",
         }]
         response = self.client.post('/anymail/sendgrid/tracking/',
                                     content_type='application/json', data=json.dumps(raw_events))
@@ -174,7 +174,7 @@ def test_deferred_event(self):
         self.assertIsInstance(event, AnymailTrackingEvent)
         self.assertEqual(event.event_type, "deferred")
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "ccf83222-0d7e-4542-8beb-893122afa757")
         self.assertEqual(event.event_id, "b_syL5UiTvWC_Ky5L6Bs5Q")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.mta_response,
@@ -187,7 +187,7 @@ def test_open_event(self):
             "ip": "66.102.6.229",
             "sg_event_id": "MjIwNDg5NTgtZGE3OC00NDI1LWFiMmMtMDUyZTU2ZmFkOTFm",
             "sg_message_id": "wrfRRvF7Q0GgwUo2CvDmEA.filter0425p1mdw1.13037.57168B4A1D.0",
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "44920b35-3e31-478b-bb67-b4f5e0c85ebc",
             "useragent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0",
             "event": "open"
         }]
@@ -200,7 +200,7 @@ def test_open_event(self):
         self.assertIsInstance(event, AnymailTrackingEvent)
         self.assertEqual(event.event_type, "opened")
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "44920b35-3e31-478b-bb67-b4f5e0c85ebc")
         self.assertEqual(event.event_id, "MjIwNDg5NTgtZGE3OC00NDI1LWFiMmMtMDUyZTU2ZmFkOTFm")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.user_agent, "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0")
@@ -211,7 +211,7 @@ def test_click_event(self):
             "sg_event_id": "OTdlOGUzYjctYjc5Zi00OWE4LWE4YWUtNjIxNjk2ZTJlNGVi",
             "sg_message_id": "_fjPjuJfRW-IPs5SuvYotg.filter0590p1mdw1.2098.57168CFC4B.0",
             "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36",
-            "smtp-id": "<[email protected]>",
+            "anymail_id": "75de5af9-a090-4325-87f9-8c599ad66f60",
             "event": "click",
             "url_offset": {"index": 0, "type": "html"},
             "email": "[email protected]",
@@ -227,7 +227,7 @@ def test_click_event(self):
         self.assertIsInstance(event, AnymailTrackingEvent)
         self.assertEqual(event.event_type, "clicked")
         self.assertEqual(event.esp_event, raw_events[0])
-        self.assertEqual(event.message_id, "<[email protected]>")
+        self.assertEqual(event.message_id, "75de5af9-a090-4325-87f9-8c599ad66f60")
         self.assertEqual(event.event_id, "OTdlOGUzYjctYjc5Zi00OWE4LWE4YWUtNjIxNjk2ZTJlNGVi")
         self.assertEqual(event.recipient, "[email protected]")
         self.assertEqual(event.user_agent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36")

tests/utils.py

@@ -4,6 +4,7 @@
 import os
 import re
 import sys
+import uuid
 import warnings
 from base64 import b64decode
 from contextlib import contextmanager
@@ -165,6 +166,14 @@ def assertEqualIgnoringHeaderFolding(self, first, second, msg=None):
         second = rfc822_unfold(second)
         self.assertEqual(first, second, msg)
 
+    def assertUUIDIsValid(self, uuid_str, version=4):
+        """Assert the uuid_str evaluates to a valid UUID"""
+        try:
+            uuid.UUID(uuid_str, version=version)
+        except (ValueError, AttributeError, TypeError):
+            return False
+        return True
+
 
 # Backported from Python 3.4
 class _AssertLogsContext(object):