Retry on git fetch errors

newsfragments/177.bugfix

@@ -0,0 +1,2 @@
+Try to avoid git fetch lock issues and sentry alerts pollution by retrying
+automatically.

src/oca_github_bot/github.py

@@ -1,6 +1,7 @@
 # Copyright (c) ACSONE SA/NV 2018-2019
 # Distributed under the MIT License (http://opensource.org/licenses/MIT).
 
+import functools
 import logging
 import os
 import shutil
@@ -13,6 +14,7 @@
 
 from . import config
 from .process import CalledProcessError, call, check_call, check_output
+from .utils import retry_on_exception
 
 _logger = logging.getLogger(__name__)
 
@@ -77,7 +79,11 @@ def temporary_clone(org, repo, branch):
         repo_url,
         "refs/heads/*:refs/heads/*",
     ]
-    check_call(fetch_cmd, cwd=repo_cache_dir)
+    retry_on_exception(
+        functools.partial(check_call, fetch_cmd, cwd=repo_cache_dir),
+        "error: cannot lock ref",
+        sleep_time=10.0,
+    )
     # check if branch exist
     branches = check_output(["git", "branch"], cwd=repo_cache_dir)
     branches = [b.strip() for b in branches.split()]

src/oca_github_bot/utils.py

@@ -1,9 +1,29 @@
 # Copyright (c) ACSONE SA/NV 2021
 # Distributed under the MIT License (http://opensource.org/licenses/MIT).
 
+import re
+import time
+
 from . import config
 
 
 def hide_secrets(s: str) -> str:
     # TODO do we want to hide other secrets ?
     return s.replace(config.GITHUB_TOKEN, "***")
+
+
+def retry_on_exception(
+    func, exception_regex: str, max_retries: int = 3, sleep_time: float = 5.0
+):
+    """Retry a function call if it raises an exception matching a regex."""
+    counter = 0
+    while True:
+        try:
+            return func()
+        except Exception as e:
+            if not re.search(exception_regex, str(e)):
+                raise
+            if counter >= max_retries:
+                raise
+            counter += 1
+            time.sleep(sleep_time)

tests/test_utils.py

@@ -1,7 +1,7 @@
 # Copyright (c) ACSONE SA/NV 2021
 # Distributed under the MIT License (http://opensource.org/licenses/MIT).
 
-from oca_github_bot.utils import hide_secrets
+from oca_github_bot.utils import hide_secrets, retry_on_exception
 
 from .common import set_config
 
@@ -16,3 +16,69 @@ def test_hide_secrets_github_token():
         assert "git push https://***@github.com" == hide_secrets(
             "git push https://[email protected]"
         )
+
+
+def test_retry_on_exception_raise_match():
+    counter = 0
+
+    def func_that_raises():
+        nonlocal counter
+        counter += 1
+        raise Exception("something")
+
+    max_retries = 2
+    sleep_time = 0.1
+
+    try:
+        retry_on_exception(
+            func_that_raises,
+            "something",
+            max_retries=max_retries,
+            sleep_time=sleep_time,
+        )
+    except Exception as e:
+        assert "something" in str(e)
+        assert counter == max_retries + 1
+
+
+def test_retry_on_exception_raise_no_match():
+    counter = 0
+
+    def func_that_raises():
+        nonlocal counter
+        counter += 1
+        raise Exception("somestuff")
+
+    max_retries = 2
+    sleep_time = 0.1
+
+    try:
+        retry_on_exception(
+            func_that_raises,
+            "something",
+            max_retries=max_retries,
+            sleep_time=sleep_time,
+        )
+    except Exception as e:
+        assert "somestuff" in str(e)
+        assert counter == 1
+
+
+def test_retry_on_exception_no_raise():
+    counter = 0
+
+    def func_that_raises():
+        nonlocal counter
+        counter += 1
+        return True
+
+    max_retries = 2
+    sleep_time = 0.1
+
+    retry_on_exception(
+        func_that_raises,
+        "something",
+        max_retries=max_retries,
+        sleep_time=sleep_time,
+    )
+    assert counter == 1

tox.ini

@@ -4,6 +4,7 @@ envlist =
   py37
   py38
   py39
+  py310
   check_readme
   pre_commit
 
@@ -13,6 +14,7 @@ python =
     3.7: py37
     3.8: py38
     3.9: py39
+    3.10: py310
 
 [testenv]
 skip_missing_interpreters = True