Closes #6590: Introduce a nightly housekeeping command to clear expired sessions and change records

Author: jeremystretch

contrib/netbox-housekeeping.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+# This shell script invokes NetBox's housekeeping management command, which
+# intended to be run nightly. This script can be copied into your system's
+# daily cron directory (e.g. /etc/cron.daily), or referenced directly from
+# within the cron configuration file.
+#
+# If NetBox has been installed into a nonstandard location, update the paths
+# below.
+/opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py housekeeping

docs/administration/housekeeping.md

@@ -0,0 +1,10 @@
+# Housekeeping
+
+NetBox includes a `housekeeping` management command that should be run nightly. This command handles:
+
+* Clearing expired authentication sessions from the database
+* Deleting changelog records older than the configured [retention time](../configuration/optional-settings.md#changelog_retention)
+
+This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be copied into your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file.
+
+The `housekeeping` command can also be run manually at any time: Running the command outside of scheduled execution times will not interfere with its operation.

docs/installation/3-netbox.md

@@ -247,6 +247,18 @@ Password (again):
 Superuser created successfully.
 ```
 
+## Schedule the Housekeeping Task
+
+NetBox includes a `housekeeping` management command that handles some recurring cleanup tasks, such as clearing out old sessions and expired change records. Although this command may be run manually, it is recommended to configure a scheduled job using the system's `cron` daemon or a similar utility.
+
+A shell script which invokes this command is included at `contrib/netbox-housekeeping.sh`. It can be copied to your system's daily cron task directory, or included within the crontab directly. (If installing NetBox into a nonstandard path, be sure to update the system paths within this script first.)
+
+```shell
+cp /opt/netbox/contrib/netbox-housekeeping.sh /etc/cron.daily/
+```
+
+See the [housekeeping documentation](../administration/housekeeping.md) for further details.
+
 ## Test the Application
 
 At this point, we should be able to run NetBox's development server for testing. We can check by starting a development instance:

docs/installation/upgrading.md

@@ -102,5 +102,12 @@ Finally, restart the gunicorn and RQ services:
 sudo systemctl restart netbox netbox-rq
 ```
 
-!!! note
-    If upgrading from an installation that uses supervisord, please see the instructions for [migrating to systemd](migrating-to-systemd.md). The use of supervisord is no longer supported.
+## Verify Housekeeping Scheduling
+
+If upgrading from a release prior to NetBox v3.0, check that a cron task (or similar scheduled process) has been configured to run NetBox's nightly housekeeping command. A shell script which invokes this command is included at `contrib/netbox-housekeeping.sh`. It can be copied to your system's daily cron task directory, or included within the crontab directly. (If NetBox has been installed in a nonstandard path, be sure to update the system paths within this script first.)
+
+```shell
+cp /opt/netbox/contrib/netbox-housekeeping.sh /etc/cron.daily/
+```
+
+See the [housekeeping documentation](../administration/housekeeping.md) for further details.

docs/release-notes/version-3.0.md

@@ -39,6 +39,7 @@ CustomValidator can also be subclassed to enforce more complex logic by overridi
 * [#4609](https://github.com/netbox-community/netbox/issues/4609) - Allow marking prefixes as fully utilized
 * [#5806](https://github.com/netbox-community/netbox/issues/5806) - Add kilometer and mile as choices for cable length unit
 * [#6154](https://github.com/netbox-community/netbox/issues/6154) - Allow decimal values for cable lengths
+* [#6590](https://github.com/netbox-community/netbox/issues/6590) - Introduce a nightly housekeeping command to clear expired sessions and change records
 
 ### Other Changes
 

mkdocs.yml

@@ -79,6 +79,7 @@ nav:
         - Developing Plugins: 'plugins/development.md'
     - Administration:
         - Permissions: 'administration/permissions.md'
+        - Housekeeping: 'administration/housekeeping.md'
         - Replicating NetBox: 'administration/replicating-netbox.md'
         - NetBox Shell: 'administration/netbox-shell.md'
     - REST API:

netbox/extras/management/commands/housekeeping.py

@@ -0,0 +1,51 @@
+from datetime import timedelta
+from importlib import import_module
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import DEFAULT_DB_ALIAS
+from django.utils import timezone
+
+from extras.models import ObjectChange
+
+
+class Command(BaseCommand):
+    help = "Perform nightly housekeeping tasks. (This command can be run at any time.)"
+
+    def handle(self, *args, **options):
+
+        # Clear expired authentication sessions (essentially replicating the `clearsessions` command)
+        self.stdout.write("[*] Clearing expired authentication sessions")
+        if options['verbosity'] >= 2:
+            self.stdout.write(f"\tConfigured session engine: {settings.SESSION_ENGINE}")
+        engine = import_module(settings.SESSION_ENGINE)
+        try:
+            engine.SessionStore.clear_expired()
+            self.stdout.write("\tSessions cleared.", self.style.SUCCESS)
+        except NotImplementedError:
+            self.stdout.write(
+                f"\tThe configured session engine ({settings.SESSION_ENGINE}) does not support "
+                f"clearing sessions; skipping."
+            )
+
+        # Delete expired ObjectRecords
+        self.stdout.write("[*] Checking for expired changelog records")
+        if settings.CHANGELOG_RETENTION:
+            cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION)
+            if options['verbosity'] >= 2:
+                self.stdout.write(f"Retention period: {settings.CHANGELOG_RETENTION} days")
+                self.stdout.write(f"\tCut-off time: {cutoff}")
+            expired_records = ObjectChange.objects.filter(time__lt=cutoff).count()
+            if expired_records:
+                self.stdout.write(f"\tDeleting {expired_records} expired records... ", self.style.WARNING, ending="")
+                self.stdout.flush()
+                ObjectChange.objects.filter(time__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS)
+                self.stdout.write("Done.", self.style.WARNING)
+            else:
+                self.stdout.write("\tNo expired records found.")
+        else:
+            self.stdout.write(
+                f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {settings.CHANGELOG_RETENTION})"
+            )
+
+        self.stdout.write("Finished.", self.style.SUCCESS)

netbox/extras/signals.py

@@ -1,13 +1,8 @@
-import random
-from datetime import timedelta
-
 from cacheops.signals import cache_invalidated, cache_read
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
-from django.db import DEFAULT_DB_ALIAS
 from django.db.models.signals import m2m_changed, post_save, pre_delete
 from django.dispatch import receiver
-from django.utils import timezone
 from django_prometheus.models import model_deletes, model_inserts, model_updates
 from prometheus_client import Counter
 
@@ -79,11 +74,6 @@ def is_same_object(instance, webhook_data):
     elif action == ObjectChangeActionChoices.ACTION_UPDATE:
         model_updates.labels(instance._meta.model_name).inc()
 
-    # Housekeeping: 0.1% chance of clearing out expired ObjectChanges
-    if settings.CHANGELOG_RETENTION and random.randint(1, 1000) == 1:
-        cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION)
-        ObjectChange.objects.filter(time__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS)
-
 
 def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs):
     """