Merge pull request #8957 from kkthxbye-code/save-job-results

Fix #8956: Save old JobResults

Author: jeremystretch

docs/administration/housekeeping.md

@@ -4,6 +4,7 @@ NetBox includes a `housekeeping` management command that should be run nightly.
 
 * Clearing expired authentication sessions from the database
 * Deleting changelog records older than the configured [retention time](../configuration/dynamic-settings.md#changelog_retention)
+* Deleting job result records older than the configured [retention time](../configuration/dynamic-settings.md#jobresult_retention)
 
 This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be linked from your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file.
 

docs/configuration/dynamic-settings.md

@@ -43,6 +43,18 @@ changes in the database indefinitely.
 
 ---
 
+## JOBRESULT_RETENTION
+
+Default: 90
+
+The number of days to retain job results (scripts and reports). Set this to `0` to retain
+job results in the database indefinitely.
+
+!!! warning
+    If enabling indefinite job results retention, it is recommended to periodically delete old entries. Otherwise, the database may eventually exceed capacity.
+
+---
+
 ## CUSTOM_VALIDATORS
 
 This is a mapping of models to [custom validators](../customization/custom-validation.md) that have been defined locally to enforce custom validation logic. An example is provided below:

netbox/extras/admin.py

@@ -40,7 +40,7 @@ class ConfigRevisionAdmin(admin.ModelAdmin):
             'fields': ('DEFAULT_USER_PREFERENCES',),
         }),
         ('Miscellaneous', {
-            'fields': ('MAINTENANCE_MODE', 'GRAPHQL_ENABLED', 'CHANGELOG_RETENTION', 'MAPS_URL'),
+            'fields': ('MAINTENANCE_MODE', 'GRAPHQL_ENABLED', 'CHANGELOG_RETENTION', 'JOBRESULT_RETENTION', 'MAPS_URL'),
         }),
         ('Config Revision', {
             'fields': ('comment',),

netbox/extras/api/views.py

@@ -179,7 +179,7 @@ def list(self, request):
             for r in JobResult.objects.filter(
                 obj_type=report_content_type,
                 status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-            ).defer('data')
+            ).order_by('name', '-created').distinct('name').defer('data')
         }
 
         # Iterate through all available Reports.
@@ -271,7 +271,7 @@ def list(self, request):
             for r in JobResult.objects.filter(
                 obj_type=script_content_type,
                 status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-            ).defer('data').order_by('created')
+            ).order_by('name', '-created').distinct('name').defer('data')
         }
 
         flat_list = []

netbox/extras/management/commands/housekeeping.py

@@ -9,6 +9,7 @@
 from django.utils import timezone
 from packaging import version
 
+from extras.models import JobResult
 from extras.models import ObjectChange
 from netbox.config import Config
 
@@ -63,6 +64,33 @@ def handle(self, *args, **options):
                 f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {config.CHANGELOG_RETENTION})"
             )
 
+        # Delete expired JobResults
+        if options['verbosity']:
+            self.stdout.write("[*] Checking for expired jobresult records")
+        if config.JOBRESULT_RETENTION:
+            cutoff = timezone.now() - timedelta(days=config.JOBRESULT_RETENTION)
+            if options['verbosity'] >= 2:
+                self.stdout.write(f"\tRetention period: {config.JOBRESULT_RETENTION} days")
+                self.stdout.write(f"\tCut-off time: {cutoff}")
+            expired_records = JobResult.objects.filter(created__lt=cutoff).count()
+            if expired_records:
+                if options['verbosity']:
+                    self.stdout.write(
+                        f"\tDeleting {expired_records} expired records... ",
+                        self.style.WARNING,
+                        ending=""
+                    )
+                    self.stdout.flush()
+                JobResult.objects.filter(created__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS)
+                if options['verbosity']:
+                    self.stdout.write("Done.", self.style.SUCCESS)
+            elif options['verbosity']:
+                self.stdout.write("\tNo expired records found.", self.style.SUCCESS)
+        elif options['verbosity']:
+            self.stdout.write(
+                f"\tSkipping: No retention period specified (JOBRESULT_RETENTION = {config.JOBRESULT_RETENTION})"
+            )
+
         # Check for new releases (if enabled)
         if options['verbosity']:
             self.stdout.write("[*] Checking for latest release")

netbox/extras/management/commands/runscript.py

@@ -113,13 +113,6 @@ def _run_script():
 
         script_content_type = ContentType.objects.get(app_label='extras', model='script')
 
-        # Delete any previous terminal state results
-        JobResult.objects.filter(
-            obj_type=script_content_type,
-            name=script.full_name,
-            status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-        ).delete()
-
         # Create the job result
         job_result = JobResult.objects.create(
             name=script.full_name,

netbox/extras/reports.py

@@ -84,15 +84,6 @@ def run_report(job_result, *args, **kwargs):
         job_result.save()
         logging.error(f"Error during execution of report {job_result.name}")
 
-    # Delete any previous terminal state results
-    JobResult.objects.filter(
-        obj_type=job_result.obj_type,
-        name=job_result.name,
-        status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-    ).exclude(
-        pk=job_result.pk
-    ).delete()
-
 
 class Report(object):
     """

netbox/extras/scripts.py

@@ -481,23 +481,14 @@ def _run_script():
     else:
         _run_script()
 
-    # Delete any previous terminal state results
-    JobResult.objects.filter(
-        obj_type=job_result.obj_type,
-        name=job_result.name,
-        status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-    ).exclude(
-        pk=job_result.pk
-    ).delete()
-
 
 def get_scripts(use_names=False):
     """
     Return a dict of dicts mapping all scripts to their modules. Set use_names to True to use each module's human-
     defined name in place of the actual module name.
     """
     scripts = OrderedDict()
-    # Iterate through all modules within the reports path. These are the user-created files in which reports are
+    # Iterate through all modules within the scripts path. These are the user-created files in which reports are
     # defined.
     for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]):
         # Remove cached module to ensure consistency with filesystem

netbox/extras/views.py

@@ -524,7 +524,7 @@ def get(self, request):
             for r in JobResult.objects.filter(
                 obj_type=report_content_type,
                 status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-            ).defer('data')
+            ).order_by('name', '-created').distinct('name').defer('data')
         }
 
         ret = []
@@ -656,7 +656,7 @@ def get(self, request):
             for r in JobResult.objects.filter(
                 obj_type=script_content_type,
                 status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-            ).defer('data')
+            ).order_by('name', '-created').distinct('name').defer('data')
         }
 
         for _scripts in scripts.values():

netbox/netbox/config/parameters.py

@@ -187,6 +187,13 @@ def __init__(self, name, label, default, description='', field=None, field_kwarg
         description="Days to retain changelog history (set to zero for unlimited)",
         field=forms.IntegerField
     ),
+    ConfigParam(
+        name='JOBRESULT_RETENTION',
+        label='Job result retention',
+        default=90,
+        description="Days to retain job result history (set to zero for unlimited)",
+        field=forms.IntegerField
+    ),
     ConfigParam(
         name='MAPS_URL',
         label='Maps URL',

netbox/netbox/views/__init__.py

@@ -19,8 +19,7 @@
 from dcim.models import (
     Cable, ConsolePort, Device, DeviceType, Interface, PowerPanel, PowerFeed, PowerPort, Rack, Site,
 )
-from extras.choices import JobResultStatusChoices
-from extras.models import ObjectChange, JobResult
+from extras.models import ObjectChange
 from extras.tables import ObjectChangeTable
 from ipam.models import Aggregate, IPAddress, IPRange, Prefix, VLAN, VRF
 from netbox.constants import SEARCH_MAX_RESULTS, SEARCH_TYPES
@@ -48,13 +47,6 @@ def get(self, request):
             pk__lt=F('_path__destination_id')
         )
 
-        # Report Results
-        report_content_type = ContentType.objects.get(app_label='extras', model='report')
-        report_results = JobResult.objects.filter(
-            obj_type=report_content_type,
-            status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-        ).defer('data')[:10]
-
         def build_stats():
             org = (
                 ("dcim.view_site", "Sites", Site.objects.restrict(request.user, 'view').count),
@@ -150,7 +142,6 @@ def build_stats():
         return render(request, self.template_name, {
             'search_form': SearchForm(),
             'stats': build_stats(),
-            'report_results': report_results,
             'changelog_table': changelog_table,
             'new_release': new_release,
         })