Skip to content

Commit 5cf31c0

Browse files
Add better autolockrenew on-failure handling capabilities. (#12307)
* autolockrenewer can now take a callback that fires when for any non-user-defined reason (e.g. not due to settlement or shutdown) a lock is lost on an auto-lock-renewed session or message. Adds tests as well and changelog notes. * add a test for receiver shutdown halting autorenewal (and corrosponding mocks) * Add proper typing and documentation to aio code. * Rename autolockrenew shutdown to close to normalize method name with other comparable instances. Adjust tests/docs/guides/etc. * Add changelog entry for the on lock renew callback. * make unused ivars truly internal (loop, executor) within autolockrenew; make tests be more precise by explicitly clearing results list between trials. * increase idle_timeout for receiveanddelete test to avoid flakiness.
1 parent fb25c20 commit 5cf31c0

18 files changed

+650
-238
lines changed

sdk/servicebus/azure-servicebus/CHANGELOG.md

+8-2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525
- Removed property `settled` on `PeekMessage`.
2626
- Removed property `expired` on `ReceivedMessage`.
2727

28+
* Add `on_lock_renew_failure` as a parameter to `AutoLockRenew.register`, taking a callback for when the lock is lost non-intentially (e.g. not via settling, shutdown, or autolockrenew duration completion)
29+
30+
**Breaking Changes**
31+
32+
* `AutoLockRenew.sleep_time` and `AutoLockRenew.renew_period` have been made internal as `_sleep_time` and `_renew_period` respectively, as it is not expected a user will have to interact with them.
33+
* `AutoLockRenew.shutdown` is now `AutoLockRenew.close` to normalize with other equivelent behaviors.
2834

2935
## 7.0.0b4 (2020-07-06)
3036

@@ -35,8 +41,8 @@
3541

3642
**BugFixes**
3743

38-
* Fixed bug where sync AutoLockRenew does not shutdown itself timely.
39-
* Fixed bug where async AutoLockRenew does not support context manager.
44+
* Fixed bug where sync `AutoLockRenew` does not shutdown itself timely.
45+
* Fixed bug where async `AutoLockRenew` does not support context manager.
4046

4147
**Breaking Changes**
4248

sdk/servicebus/azure-servicebus/README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ connstr = os.environ['SERVICE_BUS_CONN_STR']
381381
queue_name = os.environ['SERVICE_BUS_QUEUE_NAME']
382382
session_id = os.environ['SERVICE_BUS_SESSION_ID']
383383

384-
# Can also be called via "with AutoLockRenew() as renewer" to automate shutdown.
384+
# Can also be called via "with AutoLockRenew() as renewer" to automate closing.
385385
renewer = AutoLockRenew()
386386
with ServiceBusClient.from_connection_string(connstr) as client:
387387
with client.get_queue_session_receiver(queue_name, session_id=session_id) as receiver:
@@ -390,7 +390,7 @@ with ServiceBusClient.from_connection_string(connstr) as client:
390390
renewer.register(msg, timeout=60)
391391
# Do your application logic here
392392
msg.complete()
393-
renewer.shutdown()
393+
renewer.close()
394394
```
395395

396396
If for any reason auto-renewal has been interrupted or failed, this can be observed via the `auto_renew_error` property on the object being renewed.

sdk/servicebus/azure-servicebus/azure/servicebus/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from ._base_handler import ServiceBusSharedKeyCredential
1717
from ._common.message import Message, BatchMessage, PeekMessage, ReceivedMessage
1818
from ._common.constants import ReceiveSettleMode, NEXT_AVAILABLE
19-
from ._common.utils import AutoLockRenew
19+
from ._common.auto_lock_renewer import AutoLockRenew
2020

2121
TransportType = constants.TransportType
2222

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# ------------------------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the MIT License. See License.txt in the project root for
4+
# license information.
5+
# -------------------------------------------------------------------------
6+
7+
import datetime
8+
import logging
9+
import threading
10+
import time
11+
from concurrent.futures import ThreadPoolExecutor
12+
from typing import TYPE_CHECKING
13+
14+
from .._servicebus_session import ServiceBusSession
15+
from ..exceptions import AutoLockRenewFailed, AutoLockRenewTimeout, ServiceBusError
16+
from .utils import renewable_start_time, utc_now
17+
18+
if TYPE_CHECKING:
19+
from typing import Callable, Union, Optional, Awaitable
20+
from .message import ReceivedMessage
21+
LockRenewFailureCallback = Callable[[Union[ServiceBusSession, ReceivedMessage],
22+
Optional[Exception]], None]
23+
24+
_log = logging.getLogger(__name__)
25+
26+
class AutoLockRenew(object):
27+
"""Auto renew locks for messages and sessions using a background thread pool.
28+
29+
:param executor: A user-specified thread pool. This cannot be combined with
30+
setting `max_workers`.
31+
:type executor: ~concurrent.futures.ThreadPoolExecutor
32+
:param max_workers: Specify the maximum workers in the thread pool. If not
33+
specified the number used will be derived from the core count of the environment.
34+
This cannot be combined with `executor`.
35+
:type max_workers: int
36+
37+
.. admonition:: Example:
38+
39+
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
40+
:start-after: [START auto_lock_renew_message_sync]
41+
:end-before: [END auto_lock_renew_message_sync]
42+
:language: python
43+
:dedent: 4
44+
:caption: Automatically renew a message lock
45+
46+
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
47+
:start-after: [START auto_lock_renew_session_sync]
48+
:end-before: [END auto_lock_renew_session_sync]
49+
:language: python
50+
:dedent: 4
51+
:caption: Automatically renew a session lock
52+
53+
"""
54+
55+
def __init__(self, executor=None, max_workers=None):
56+
self._executor = executor or ThreadPoolExecutor(max_workers=max_workers)
57+
self._shutdown = threading.Event()
58+
self._sleep_time = 1
59+
self._renew_period = 10
60+
61+
def __enter__(self):
62+
if self._shutdown.is_set():
63+
raise ServiceBusError("The AutoLockRenew has already been shutdown. Please create a new instance for"
64+
" auto lock renewing.")
65+
return self
66+
67+
def __exit__(self, *args):
68+
self.close()
69+
70+
def _renewable(self, renewable):
71+
# pylint: disable=protected-access
72+
if self._shutdown.is_set():
73+
return False
74+
if hasattr(renewable, '_settled') and renewable._settled:
75+
return False
76+
if not renewable._receiver._running:
77+
return False
78+
if renewable._lock_expired:
79+
return False
80+
return True
81+
82+
def _auto_lock_renew(self, renewable, starttime, timeout, on_lock_renew_failure=None):
83+
# pylint: disable=protected-access
84+
_log.debug("Running lock auto-renew thread for %r seconds", timeout)
85+
error = None
86+
clean_shutdown = False # Only trigger the on_lock_renew_failure if halting was not expected (shutdown, etc)
87+
try:
88+
while self._renewable(renewable):
89+
if (utc_now() - starttime) >= datetime.timedelta(seconds=timeout):
90+
_log.debug("Reached auto lock renew timeout - letting lock expire.")
91+
raise AutoLockRenewTimeout("Auto-renew period ({} seconds) elapsed.".format(timeout))
92+
if (renewable.locked_until_utc - utc_now()) <= datetime.timedelta(seconds=self._renew_period):
93+
_log.debug("%r seconds or less until lock expires - auto renewing.", self._renew_period)
94+
renewable.renew_lock()
95+
time.sleep(self._sleep_time)
96+
clean_shutdown = not renewable._lock_expired
97+
except AutoLockRenewTimeout as e:
98+
error = e
99+
renewable.auto_renew_error = e
100+
clean_shutdown = not renewable._lock_expired
101+
except Exception as e: # pylint: disable=broad-except
102+
_log.debug("Failed to auto-renew lock: %r. Closing thread.", e)
103+
error = AutoLockRenewFailed(
104+
"Failed to auto-renew lock",
105+
inner_exception=e)
106+
renewable.auto_renew_error = error
107+
finally:
108+
if on_lock_renew_failure and not clean_shutdown:
109+
on_lock_renew_failure(renewable, error)
110+
111+
def register(self, renewable, timeout=300, on_lock_renew_failure=None):
112+
"""Register a renewable entity for automatic lock renewal.
113+
114+
:param renewable: A locked entity that needs to be renewed.
115+
:type renewable: ~azure.servicebus.ReceivedMessage or
116+
~azure.servicebus.ServiceBusSession
117+
:param float timeout: A time in seconds that the lock should be maintained for.
118+
Default value is 300 (5 minutes).
119+
:param Optional[LockRenewFailureCallback] on_lock_renew_failure:
120+
A callback may be specified to be called when the lock is lost on the renewable that is being registered.
121+
Default value is None (no callback).
122+
"""
123+
if self._shutdown.is_set():
124+
raise ServiceBusError("The AutoLockRenew has already been shutdown. Please create a new instance for"
125+
" auto lock renewing.")
126+
starttime = renewable_start_time(renewable)
127+
self._executor.submit(self._auto_lock_renew, renewable, starttime, timeout, on_lock_renew_failure)
128+
129+
def close(self, wait=True):
130+
"""Cease autorenewal by shutting down the thread pool to clean up any remaining lock renewal threads.
131+
132+
:param wait: Whether to block until thread pool has shutdown. Default is `True`.
133+
:type wait: bool
134+
"""
135+
self._shutdown.set()
136+
self._executor.shutdown(wait=wait)

sdk/servicebus/azure-servicebus/azure/servicebus/_common/utils.py

+1-102
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,17 @@
77
import sys
88
import datetime
99
import logging
10-
import threading
11-
import time
1210
import functools
1311
import platform
1412
from typing import Optional, Dict
1513
try:
1614
from urlparse import urlparse
1715
except ImportError:
1816
from urllib.parse import urlparse
19-
from concurrent.futures import ThreadPoolExecutor
2017

2118
from uamqp import authentication, types
2219

23-
from ..exceptions import AutoLockRenewFailed, AutoLockRenewTimeout, ServiceBusError
20+
from ..exceptions import ServiceBusError
2421
from .._version import VERSION
2522
from .constants import (
2623
JWT_TOKEN_SCOPE,
@@ -180,101 +177,3 @@ def generate_dead_letter_entity_name(
180177
)
181178

182179
return entity_name
183-
184-
185-
class AutoLockRenew(object):
186-
"""Auto renew locks for messages and sessions using a background thread pool.
187-
188-
:param executor: A user-specified thread pool. This cannot be combined with
189-
setting `max_workers`.
190-
:type executor: ~concurrent.futures.ThreadPoolExecutor
191-
:param max_workers: Specify the maximum workers in the thread pool. If not
192-
specified the number used will be derived from the core count of the environment.
193-
This cannot be combined with `executor`.
194-
:type max_workers: int
195-
196-
.. admonition:: Example:
197-
198-
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
199-
:start-after: [START auto_lock_renew_message_sync]
200-
:end-before: [END auto_lock_renew_message_sync]
201-
:language: python
202-
:dedent: 4
203-
:caption: Automatically renew a message lock
204-
205-
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
206-
:start-after: [START auto_lock_renew_session_sync]
207-
:end-before: [END auto_lock_renew_session_sync]
208-
:language: python
209-
:dedent: 4
210-
:caption: Automatically renew a session lock
211-
212-
"""
213-
214-
def __init__(self, executor=None, max_workers=None):
215-
self.executor = executor or ThreadPoolExecutor(max_workers=max_workers)
216-
self._shutdown = threading.Event()
217-
self.sleep_time = 1
218-
self.renew_period = 10
219-
220-
def __enter__(self):
221-
if self._shutdown.is_set():
222-
raise ServiceBusError("The AutoLockRenew has already been shutdown. Please create a new instance for"
223-
" auto lock renewing.")
224-
return self
225-
226-
def __exit__(self, *args):
227-
self.shutdown()
228-
229-
def _renewable(self, renewable):
230-
if self._shutdown.is_set():
231-
return False
232-
if hasattr(renewable, '_settled') and renewable._settled: # pylint: disable=protected-access
233-
return False
234-
if renewable._lock_expired: # pylint: disable=protected-access
235-
return False
236-
return True
237-
238-
def _auto_lock_renew(self, renewable, starttime, timeout):
239-
_log.debug("Running lock auto-renew thread for %r seconds", timeout)
240-
try:
241-
while self._renewable(renewable):
242-
if (utc_now() - starttime) >= datetime.timedelta(seconds=timeout):
243-
_log.debug("Reached auto lock renew timeout - letting lock expire.")
244-
raise AutoLockRenewTimeout("Auto-renew period ({} seconds) elapsed.".format(timeout))
245-
if (renewable.locked_until_utc - utc_now()) <= datetime.timedelta(seconds=self.renew_period):
246-
_log.debug("%r seconds or less until lock expires - auto renewing.", self.renew_period)
247-
renewable.renew_lock()
248-
time.sleep(self.sleep_time)
249-
except AutoLockRenewTimeout as e:
250-
renewable.auto_renew_error = e
251-
except Exception as e: # pylint: disable=broad-except
252-
_log.debug("Failed to auto-renew lock: %r. Closing thread.", e)
253-
error = AutoLockRenewFailed(
254-
"Failed to auto-renew lock",
255-
inner_exception=e)
256-
renewable.auto_renew_error = error
257-
258-
def register(self, renewable, timeout=300):
259-
"""Register a renewable entity for automatic lock renewal.
260-
261-
:param renewable: A locked entity that needs to be renewed.
262-
:type renewable: ~azure.servicebus.ReceivedMessage or
263-
~azure.servicebus.Session
264-
:param float timeout: A time in seconds that the lock should be maintained for.
265-
Default value is 300 (5 minutes).
266-
"""
267-
if self._shutdown.is_set():
268-
raise ServiceBusError("The AutoLockRenew has already been shutdown. Please create a new instance for"
269-
" auto lock renewing.")
270-
starttime = renewable_start_time(renewable)
271-
self.executor.submit(self._auto_lock_renew, renewable, starttime, timeout)
272-
273-
def shutdown(self, wait=True):
274-
"""Shutdown the thread pool to clean up any remaining lock renewal threads.
275-
276-
:param wait: Whether to block until thread pool has shutdown. Default is `True`.
277-
:type wait: bool
278-
"""
279-
self._shutdown.set()
280-
self.executor.shutdown(wait=wait)

sdk/servicebus/azure-servicebus/azure/servicebus/aio/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ._servicebus_session_receiver_async import ServiceBusSessionReceiver
1111
from ._servicebus_session_async import ServiceBusSession
1212
from ._servicebus_client_async import ServiceBusClient
13-
from ._async_utils import AutoLockRenew
13+
from ._async_auto_lock_renewer import AutoLockRenew
1414

1515
__all__ = [
1616
'ReceivedMessage',

0 commit comments

Comments
 (0)