Closes #11558: Add support for remote data sources (#11646)

* WIP

* WIP

* Add git sync

* Fix file hashing

* Add last_synced to DataSource

* Build out UI & API resources

* Add status field to DataSource

* Add UI control to sync data source

* Add API endpoint to sync data sources

* Fix display of DataSource job results

* DataSource password should be write-only

* General cleanup

* Add data file UI view

* Punt on HTTP, FTP support for now

* Add DataSource URL validation

* Add HTTP proxy support to git fetcher

* Add management command to sync data sources

* DataFile REST API endpoints should be read-only

* Refactor fetch methods into backend classes

* Replace auth & git branch fields with general-purpose parameters

* Fix last_synced time

* Render discrete form fields for backend parameters

* Enable dynamic edit form for DataSource

* Register DataBackend classes in application registry

* Add search indexers for DataSource, DataFile

* Add single & bulk delete views for DataFile

* Add model documentation

* Convert DataSource to a primary model

* Introduce pre_sync & post_sync signals

* Clean up migrations

* Rename url to source_url

* Clean up filtersets

* Add API & filterset tests

* Add view tests

* Add initSelect() to HTMX refresh handler

* Render DataSourceForm fieldsets dynamically

* Update compiled static resources

Author: jeremystretch

docs/models/core/datafile.md

@@ -0,0 +1,25 @@
+# Data Files
+
+A data file object is the representation in NetBox's database of some file belonging to a remote [data source](./datasource.md). Data files are synchronized automatically, and cannot be modified locally (although they can be deleted).
+
+## Fields
+
+### Source
+
+The [data source](./datasource.md) to which this file belongs.
+
+### Path
+
+The path to the file, relative to its source's URL. For example, a file at `/opt/config-data/routing/bgp/peer.yaml` with a source URL of `file:///opt/config-data/` would have its path set to `routing/bgp/peer.yaml`.
+
+### Last Updated
+
+The date and time at which the file most recently updated from its source. Note that this attribute is updated only when the file's contents have been modified. Re-synchronizing the data source will not update this timestamp if the upstream file's data has not changed.
+
+### Size
+
+The file's size, in bytes.
+
+### Hash
+
+A [SHA256 hash](https://en.wikipedia.org/wiki/SHA-2) of the file's data. This can be compared to a hash taken from the original file to determine whether any changes have been made.

docs/models/core/datasource.md

@@ -0,0 +1,47 @@
+# Data Sources
+
+A data source represents some external repository of data which NetBox can consume, such as a git repository. Files within the data source are synchronized to NetBox by saving them in the database as [data file](./datafile.md) objects.
+
+## Fields
+
+### Name
+
+The data source's human-friendly name.
+
+### Type
+
+The type of data source. Supported options include:
+
+* Local directory
+* git repository
+
+### URL
+
+The URL identifying the remote source. Some examples are included below.
+
+| Type | Example URL |
+|------|-------------|
+| Local | file:///var/my/data/source/ |
+| git | https://https://github.com/my-organization/my-repo |
+
+### Status
+
+The source's current synchronization status. Note that this cannot be set manually: It is updated automatically when the source is synchronized.
+
+### Enabled
+
+If false, synchronization will be disabled.
+
+### Ignore Rules
+
+A set of rules (one per line) identifying filenames to ignore during synchronization. Some examples are provided below. See Python's [`fnmatch()` documentation](https://docs.python.org/3/library/fnmatch.html) for a complete reference.
+
+| Rule           | Description                              |
+|----------------|------------------------------------------|
+| `README`       | Ignore any files named `README`          |
+| `*.txt`        | Ignore any files with a `.txt` extension |
+| `data???.json` | Ignore e.g. `data123.json`               |
+
+### Last Synced
+
+The date and time at which the source was most recently synchronized successfully.

netbox/core/__init__.py

@@ -0,0 +1,25 @@
+from rest_framework import serializers
+
+from core.models import *
+from netbox.api.serializers import WritableNestedSerializer
+
+__all__ = [
+    'NestedDataFileSerializer',
+    'NestedDataSourceSerializer',
+]
+
+
+class NestedDataSourceSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datasource-detail')
+
+    class Meta:
+        model = DataSource
+        fields = ['id', 'url', 'display', 'name']
+
+
+class NestedDataFileSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datafile-detail')
+
+    class Meta:
+        model = DataFile
+        fields = ['id', 'url', 'display', 'path']

netbox/core/api/__init__.py

@@ -0,0 +1,51 @@
+from rest_framework import serializers
+
+from core.choices import *
+from core.models import *
+from netbox.api.fields import ChoiceField
+from netbox.api.serializers import NetBoxModelSerializer
+from .nested_serializers import *
+
+__all__ = (
+    'DataSourceSerializer',
+)
+
+
+class DataSourceSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(
+        view_name='core-api:datasource-detail'
+    )
+    type = ChoiceField(
+        choices=DataSourceTypeChoices
+    )
+    status = ChoiceField(
+        choices=DataSourceStatusChoices,
+        read_only=True
+    )
+
+    # Related object counts
+    file_count = serializers.IntegerField(
+        read_only=True
+    )
+
+    class Meta:
+        model = DataSource
+        fields = [
+            'id', 'url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description', 'comments',
+            'parameters', 'ignore_rules', 'created', 'last_updated', 'file_count',
+        ]
+
+
+class DataFileSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(
+        view_name='core-api:datafile-detail'
+    )
+    source = NestedDataSourceSerializer(
+        read_only=True
+    )
+
+    class Meta:
+        model = DataFile
+        fields = [
+            'id', 'url', 'display', 'source', 'path', 'last_updated', 'size', 'hash',
+        ]

netbox/core/api/nested_serializers.py

@@ -0,0 +1,13 @@
+from netbox.api.routers import NetBoxRouter
+from . import views
+
+
+router = NetBoxRouter()
+router.APIRootView = views.CoreRootView
+
+# Data sources
+router.register('data-sources', views.DataSourceViewSet)
+router.register('data-files', views.DataFileViewSet)
+
+app_name = 'core-api'
+urlpatterns = router.urls

netbox/core/api/serializers.py

@@ -0,0 +1,52 @@
+from django.shortcuts import get_object_or_404
+
+from rest_framework.decorators import action
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
+from rest_framework.routers import APIRootView
+
+from core import filtersets
+from core.models import *
+from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet
+from utilities.utils import count_related
+from . import serializers
+
+
+class CoreRootView(APIRootView):
+    """
+    Core API root view
+    """
+    def get_view_name(self):
+        return 'Core'
+
+
+#
+# Data sources
+#
+
+class DataSourceViewSet(NetBoxModelViewSet):
+    queryset = DataSource.objects.annotate(
+        file_count=count_related(DataFile, 'source')
+    )
+    serializer_class = serializers.DataSourceSerializer
+    filterset_class = filtersets.DataSourceFilterSet
+
+    @action(detail=True, methods=['post'])
+    def sync(self, request, pk):
+        """
+        Enqueue a job to synchronize the DataSource.
+        """
+        if not request.user.has_perm('extras.sync_datasource'):
+            raise PermissionDenied("Syncing data sources requires the core.sync_datasource permission.")
+
+        datasource = get_object_or_404(DataSource, pk=pk)
+        datasource.enqueue_sync_job(request)
+        serializer = serializers.DataSourceSerializer(datasource, context={'request': request})
+
+        return Response(serializer.data)
+
+
+class DataFileViewSet(NetBoxReadOnlyModelViewSet):
+    queryset = DataFile.objects.defer('data').prefetch_related('source')
+    serializer_class = serializers.DataFileSerializer
+    filterset_class = filtersets.DataFileFilterSet

netbox/core/api/urls.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    name = "core"
+
+    def ready(self):
+        from . import data_backends, search

netbox/core/api/views.py

@@ -0,0 +1,34 @@
+from django.utils.translation import gettext as _
+
+from utilities.choices import ChoiceSet
+
+
+#
+# Data sources
+#
+
+class DataSourceTypeChoices(ChoiceSet):
+    LOCAL = 'local'
+    GIT = 'git'
+
+    CHOICES = (
+        (LOCAL, _('Local'), 'gray'),
+        (GIT, _('Git'), 'blue'),
+    )
+
+
+class DataSourceStatusChoices(ChoiceSet):
+
+    NEW = 'new'
+    QUEUED = 'queued'
+    SYNCING = 'syncing'
+    COMPLETED = 'completed'
+    FAILED = 'failed'
+
+    CHOICES = (
+        (NEW, _('New'), 'blue'),
+        (QUEUED, _('Queued'), 'orange'),
+        (SYNCING, _('Syncing'), 'cyan'),
+        (COMPLETED, _('Completed'), 'green'),
+        (FAILED, _('Failed'), 'red'),
+    )

netbox/core/apps.py

@@ -0,0 +1,117 @@
+import logging
+import subprocess
+import tempfile
+from contextlib import contextmanager
+from urllib.parse import quote, urlunparse, urlparse
+
+from django import forms
+from django.conf import settings
+from django.utils.translation import gettext as _
+
+from netbox.registry import registry
+from .choices import DataSourceTypeChoices
+from .exceptions import SyncError
+
+__all__ = (
+    'LocalBackend',
+    'GitBackend',
+)
+
+logger = logging.getLogger('netbox.data_backends')
+
+
+def register_backend(name):
+    """
+    Decorator for registering a DataBackend class.
+    """
+    def _wrapper(cls):
+        registry['data_backends'][name] = cls
+        return cls
+
+    return _wrapper
+
+
+class DataBackend:
+    parameters = {}
+
+    def __init__(self, url, **kwargs):
+        self.url = url
+        self.params = kwargs
+
+    @property
+    def url_scheme(self):
+        return urlparse(self.url).scheme.lower()
+
+    @contextmanager
+    def fetch(self):
+        raise NotImplemented()
+
+
+@register_backend(DataSourceTypeChoices.LOCAL)
+class LocalBackend(DataBackend):
+
+    @contextmanager
+    def fetch(self):
+        logger.debug(f"Data source type is local; skipping fetch")
+        local_path = urlparse(self.url).path  # Strip file:// scheme
+
+        yield local_path
+
+
+@register_backend(DataSourceTypeChoices.GIT)
+class GitBackend(DataBackend):
+    parameters = {
+        'username': forms.CharField(
+            required=False,
+            label=_('Username'),
+            widget=forms.TextInput(attrs={'class': 'form-control'})
+        ),
+        'password': forms.CharField(
+            required=False,
+            label=_('Password'),
+            widget=forms.TextInput(attrs={'class': 'form-control'})
+        ),
+        'branch': forms.CharField(
+            required=False,
+            label=_('Branch'),
+            widget=forms.TextInput(attrs={'class': 'form-control'})
+        )
+    }
+
+    @contextmanager
+    def fetch(self):
+        local_path = tempfile.TemporaryDirectory()
+
+        # Add authentication credentials to URL (if specified)
+        username = self.params.get('username')
+        password = self.params.get('password')
+        if username and password:
+            url_components = list(urlparse(self.url))
+            # Prepend username & password to netloc
+            url_components[1] = quote(f'{username}@{password}:') + url_components[1]
+            url = urlunparse(url_components)
+        else:
+            url = self.url
+
+        # Compile git arguments
+        args = ['git', 'clone', '--depth', '1']
+        if branch := self.params.get('branch'):
+            args.extend(['--branch', branch])
+        args.extend([url, local_path.name])
+
+        # Prep environment variables
+        env_vars = {}
+        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
+            env_vars['http_proxy'] = settings.HTTP_PROXIES.get(self.url_scheme)
+
+        logger.debug(f"Cloning git repo: {' '.join(args)}")
+        try:
+            subprocess.run(args, check=True, capture_output=True, env=env_vars)
+        except subprocess.CalledProcessError as e:
+            raise SyncError(
+                f"Fetching remote data failed: {e.stderr}"
+            )
+
+        yield local_path.name
+
+        local_path.cleanup()

netbox/core/choices.py

@@ -0,0 +1,2 @@
+class SyncError(Exception):
+    pass