Closes #13381: Enable plugins to register custom data backends (#14095)

* Initial work on #13381

* Fix backend type display in table column

* Fix data source type choices during bulk edit

* Misc cleanup

* Move backend utils from core app to netbox

* Move backend type validation from serializer to model

Author: jeremystretch

docs/plugins/development/data-backends.md

@@ -0,0 +1,23 @@
+# Data Backends
+
+[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.
+
+```python title="data_backends.py"
+from netbox.data_backends import DataBackend
+
+class MyDataBackend(DataBackend):
+    name = 'mybackend'
+    label = 'My Backend'
+    ...
+```
+
+To register one or more data backends with NetBox, define a list named `backends` at the end of this file:
+
+```python title="data_backends.py"
+backends = [MyDataBackend]
+```
+
+!!! tip
+    The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.
+
+::: core.data_backends.DataBackend

docs/plugins/development/index.md

@@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
 | `middleware`          | A list of middleware classes to append after NetBox's build-in middleware                                                |
 | `queues`              | A list of custom background task queues to create                                                                        |
 | `search_extensions`   | The dotted path to the list of search index classes (default: `search.indexes`)                                          |
+| `data_backends`       | The dotted path to the list of data source backend classes (default: `data_backends.backends`)                           |
 | `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`)              |
 | `menu_items`          | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`)                      |
 | `graphql_schema`      | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`)                                 |

mkdocs.yml

@@ -136,6 +136,7 @@ nav:
             - Forms: 'plugins/development/forms.md'
             - Filters & Filter Sets: 'plugins/development/filtersets.md'
             - Search: 'plugins/development/search.md'
+            - Data Backends: 'plugins/development/data-backends.md'
             - REST API: 'plugins/development/rest-api.md'
             - GraphQL API: 'plugins/development/graphql-api.md'
             - Background Tasks: 'plugins/development/background-tasks.md'

netbox/core/api/serializers.py

@@ -4,6 +4,7 @@
 from core.models import *
 from netbox.api.fields import ChoiceField, ContentTypeField
 from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
+from netbox.utils import get_data_backend_choices
 from users.api.nested_serializers import NestedUserSerializer
 from .nested_serializers import *
 
@@ -19,7 +20,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
         view_name='core-api:datasource-detail'
     )
     type = ChoiceField(
-        choices=DataSourceTypeChoices
+        choices=get_data_backend_choices()
     )
     status = ChoiceField(
         choices=DataSourceStatusChoices,

netbox/core/choices.py

@@ -7,18 +7,6 @@
 # Data sources
 #
 
-class DataSourceTypeChoices(ChoiceSet):
-    LOCAL = 'local'
-    GIT = 'git'
-    AMAZON_S3 = 'amazon-s3'
-
-    CHOICES = (
-        (LOCAL, _('Local'), 'gray'),
-        (GIT, 'Git', 'blue'),
-        (AMAZON_S3, 'Amazon S3', 'blue'),
-    )
-
-
 class DataSourceStatusChoices(ChoiceSet):
     NEW = 'new'
     QUEUED = 'queued'

netbox/core/data_backends.py

@@ -10,61 +10,24 @@
 from django.conf import settings
 from django.utils.translation import gettext as _
 
-from netbox.registry import registry
-from .choices import DataSourceTypeChoices
+from netbox.data_backends import DataBackend
+from netbox.utils import register_data_backend
 from .exceptions import SyncError
 
 __all__ = (
-    'LocalBackend',
     'GitBackend',
+    'LocalBackend',
     'S3Backend',
 )
 
 logger = logging.getLogger('netbox.data_backends')
 
 
-def register_backend(name):
-    """
-    Decorator for registering a DataBackend class.
-    """
-
-    def _wrapper(cls):
-        registry['data_backends'][name] = cls
-        return cls
-
-    return _wrapper
-
-
-class DataBackend:
-    parameters = {}
-    sensitive_parameters = []
-
-    # Prevent Django's template engine from calling the backend
-    # class when referenced via DataSource.backend_class
-    do_not_call_in_templates = True
-
-    def __init__(self, url, **kwargs):
-        self.url = url
-        self.params = kwargs
-        self.config = self.init_config()
-
-    def init_config(self):
-        """
-        Hook to initialize the instance's configuration.
-        """
-        return
-
-    @property
-    def url_scheme(self):
-        return urlparse(self.url).scheme.lower()
-
-    @contextmanager
-    def fetch(self):
-        raise NotImplemented()
-
-
-@register_backend(DataSourceTypeChoices.LOCAL)
+@register_data_backend()
 class LocalBackend(DataBackend):
+    name = 'local'
+    label = _('Local')
+    is_local = True
 
     @contextmanager
     def fetch(self):
@@ -74,8 +37,10 @@ def fetch(self):
         yield local_path
 
 
-@register_backend(DataSourceTypeChoices.GIT)
+@register_data_backend()
 class GitBackend(DataBackend):
+    name = 'git'
+    label = 'Git'
     parameters = {
         'username': forms.CharField(
             required=False,
@@ -144,8 +109,10 @@ def fetch(self):
         local_path.cleanup()
 
 
-@register_backend(DataSourceTypeChoices.AMAZON_S3)
+@register_data_backend()
 class S3Backend(DataBackend):
+    name = 'amazon-s3'
+    label = 'Amazon S3'
     parameters = {
         'aws_access_key_id': forms.CharField(
             label=_('AWS access key ID'),

netbox/core/filtersets.py

@@ -4,6 +4,7 @@
 import django_filters
 
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
+from netbox.utils import get_data_backend_choices
 from .choices import *
 from .models import *
 
@@ -16,7 +17,7 @@
 
 class DataSourceFilterSet(NetBoxModelFilterSet):
     type = django_filters.MultipleChoiceFilter(
-        choices=DataSourceTypeChoices,
+        choices=get_data_backend_choices,
         null_value=None
     )
     status = django_filters.MultipleChoiceFilter(

netbox/core/forms/bulk_edit.py

@@ -1,10 +1,9 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
 
-from core.choices import DataSourceTypeChoices
 from core.models import *
 from netbox.forms import NetBoxModelBulkEditForm
-from utilities.forms import add_blank_choice
+from netbox.utils import get_data_backend_choices
 from utilities.forms.fields import CommentField
 from utilities.forms.widgets import BulkEditNullBooleanSelect
 
@@ -16,9 +15,8 @@
 class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
     type = forms.ChoiceField(
         label=_('Type'),
-        choices=add_blank_choice(DataSourceTypeChoices),
-        required=False,
-        initial=''
+        choices=get_data_backend_choices,
+        required=False
     )
     enabled = forms.NullBooleanField(
         required=False,

netbox/core/forms/filtersets.py

@@ -8,6 +8,7 @@
 from extras.forms.mixins import SavedFiltersMixin
 from extras.utils import FeatureQuery
 from netbox.forms import NetBoxModelFilterSetForm
+from netbox.utils import get_data_backend_choices
 from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
 from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
 from utilities.forms.widgets import APISelectMultiple, DateTimePicker
@@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
     )
     type = forms.MultipleChoiceField(
         label=_('Type'),
-        choices=DataSourceTypeChoices,
+        choices=get_data_backend_choices,
         required=False
     )
     status = forms.MultipleChoiceField(

netbox/core/forms/model_forms.py

@@ -7,6 +7,7 @@
 from core.models import *
 from netbox.forms import NetBoxModelForm
 from netbox.registry import registry
+from netbox.utils import get_data_backend_choices
 from utilities.forms import get_field_value
 from utilities.forms.fields import CommentField
 from utilities.forms.widgets import HTMXSelect
@@ -18,6 +19,10 @@
 
 
 class DataSourceForm(NetBoxModelForm):
+    type = forms.ChoiceField(
+        choices=get_data_backend_choices,
+        widget=HTMXSelect()
+    )
     comments = CommentField()
 
     class Meta:
@@ -26,7 +31,6 @@ class Meta:
             'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
         ]
         widgets = {
-            'type': HTMXSelect(),
             'ignore_rules': forms.Textarea(
                 attrs={
                     'rows': 5,
@@ -56,12 +60,13 @@ def __init__(self, *args, **kwargs):
 
         # Add backend-specific form fields
         self.backend_fields = []
-        for name, form_field in backend.parameters.items():
-            field_name = f'backend_{name}'
-            self.backend_fields.append(field_name)
-            self.fields[field_name] = copy.copy(form_field)
-            if self.instance and self.instance.parameters:
-                self.fields[field_name].initial = self.instance.parameters.get(name)
+        if backend:
+            for name, form_field in backend.parameters.items():
+                field_name = f'backend_{name}'
+                self.backend_fields.append(field_name)
+                self.fields[field_name] = copy.copy(form_field)
+                if self.instance and self.instance.parameters:
+                    self.fields[field_name].initial = self.instance.parameters.get(name)
 
     def save(self, *args, **kwargs):
 

netbox/core/migrations/0006_datasource_type_remove_choices.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.6 on 2023-10-20 17:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_job_created_auto_now'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasource',
+            name='type',
+            field=models.CharField(max_length=50),
+        ),
+    ]

netbox/core/models/data.py

@@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
     )
     type = models.CharField(
         verbose_name=_('type'),
-        max_length=50,
-        choices=DataSourceTypeChoices,
-        default=DataSourceTypeChoices.LOCAL
+        max_length=50
     )
     source_url = models.CharField(
         max_length=200,
@@ -96,8 +94,9 @@ def get_absolute_url(self):
     def docs_url(self):
         return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
 
-    def get_type_color(self):
-        return DataSourceTypeChoices.colors.get(self.type)
+    def get_type_display(self):
+        if backend := registry['data_backends'].get(self.type):
+            return backend.label
 
     def get_status_color(self):
         return DataSourceStatusChoices.colors.get(self.status)
@@ -110,10 +109,6 @@ def url_scheme(self):
     def backend_class(self):
         return registry['data_backends'].get(self.type)
 
-    @property
-    def is_local(self):
-        return self.type == DataSourceTypeChoices.LOCAL
-
     @property
     def ready_for_sync(self):
         return self.enabled and self.status not in (
@@ -123,8 +118,14 @@ def ready_for_sync(self):
 
     def clean(self):
 
+        # Validate data backend type
+        if self.type and self.type not in registry['data_backends']:
+            raise ValidationError({
+                'type': _("Unknown backend type: {type}".format(type=self.type))
+            })
+
         # Ensure URL scheme matches selected type
-        if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''):
+        if self.backend_class.is_local and self.url_scheme not in ('file', ''):
             raise ValidationError({
                 'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
             })

netbox/core/tables/columns.py

@@ -0,0 +1,20 @@
+import django_tables2 as tables
+
+from netbox.registry import registry
+
+__all__ = (
+    'BackendTypeColumn',
+)
+
+
+class BackendTypeColumn(tables.Column):
+    """
+    Display a data backend type.
+    """
+    def render(self, value):
+        if backend := registry['data_backends'].get(value):
+            return backend.label
+        return value
+
+    def value(self, value):
+        return value

netbox/core/tables/data.py

@@ -3,6 +3,7 @@
 
 from core.models import *
 from netbox.tables import NetBoxTable, columns
+from .columns import BackendTypeColumn
 
 __all__ = (
     'DataFileTable',
@@ -15,8 +16,8 @@ class DataSourceTable(NetBoxTable):
         verbose_name=_('Name'),
         linkify=True
     )
-    type = columns.ChoiceFieldColumn(
-        verbose_name=_('Type'),
+    type = BackendTypeColumn(
+        verbose_name=_('Type')
     )
     status = columns.ChoiceFieldColumn(
         verbose_name=_('Status'),
@@ -34,8 +35,8 @@ class DataSourceTable(NetBoxTable):
     class Meta(NetBoxTable.Meta):
         model = DataSource
         fields = (
-            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created',
-            'last_updated', 'file_count',
+            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
+            'created', 'last_updated', 'file_count',
         )
         default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')