diff --git a/docs/configuration.md b/docs/configuration.md index bcf72e3..ec333ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -58,7 +58,15 @@ Note that a valid prefix is required, as the randomly-generated branch ID alone Default: `[]` (empty list) -A list of import paths to functions which validate whether a branch is permitted to be synced. +A list of import paths to functions which validate whether a branch is permitted to be synced from main. + +--- + +## `pull_validators` + +Default: `[]` (empty list) + +A list of import paths to functions which validate whether changes from other branches can be pulled into a branch. --- @@ -66,7 +74,7 @@ A list of import paths to functions which validate whether a branch is permitted Default: `[]` (empty list) -A list of import paths to functions which validate whether a branch is permitted to be merged. +A list of import paths to functions which validate whether a branch is permitted to be merged into main. --- @@ -74,7 +82,7 @@ A list of import paths to functions which validate whether a branch is permitted Default: `[]` (empty list) -A list of import paths to functions which validate whether a branch is permitted to be reverted. +A list of import paths to functions which validate whether a previously merged branch is permitted to be reverted. --- diff --git a/docs/models/branchevent.md b/docs/models/branchevent.md index 6180b2b..b08853b 100644 --- a/docs/models/branchevent.md +++ b/docs/models/branchevent.md @@ -12,6 +12,10 @@ The time at which the event occurred. The [branch](./branch.md) to which this event pertains. +### Related Branch + +The related branch affected by this event, where applicable. (This is relevant only when one branch is merged into another.) + ### User The NetBox user responsible for triggering this event. This field may be null if the event was triggered by an internal process. diff --git a/netbox_branching/__init__.py b/netbox_branching/__init__.py index 34d5d08..b52851f 100644 --- a/netbox_branching/__init__.py +++ b/netbox_branching/__init__.py @@ -33,6 +33,7 @@ class AppConfig(PluginConfig): # Branch action validators 'sync_validators': [], + 'pull_validators': [], 'merge_validators': [], 'revert_validators': [], 'archive_validators': [], diff --git a/netbox_branching/api/serializers.py b/netbox_branching/api/serializers.py index d7b24c6..79fdc8f 100644 --- a/netbox_branching/api/serializers.py +++ b/netbox_branching/api/serializers.py @@ -1,6 +1,7 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers +from core.api.serializers import ObjectChangeSerializer from core.choices import ObjectChangeActionChoices from netbox.api.exceptions import SerializerNotFound from netbox.api.fields import ChoiceField, ContentTypeField @@ -13,6 +14,7 @@ __all__ = ( 'BranchSerializer', 'BranchEventSerializer', + 'BranchPullSerializer', 'ChangeDiffSerializer', 'CommitSerializer', ) @@ -58,6 +60,11 @@ class BranchEventSerializer(NetBoxModelSerializer): nested=True, read_only=True ) + related_branch = BranchSerializer( + nested=True, + read_only=True, + allow_null=True + ) user = UserSerializer( nested=True, read_only=True @@ -70,7 +77,7 @@ class BranchEventSerializer(NetBoxModelSerializer): class Meta: model = BranchEvent fields = [ - 'id', 'url', 'display', 'time', 'branch', 'user', 'type', + 'id', 'url', 'display', 'time', 'branch', 'related_branch', 'user', 'type', ] brief_fields = ('id', 'url', 'display') @@ -138,4 +145,29 @@ def get_object(self, obj): class CommitSerializer(serializers.Serializer): - commit = serializers.BooleanField(required=False) + commit = serializers.BooleanField( + required=False, + default=False + ) + + +class BranchPullSerializer(CommitSerializer): + source = BranchSerializer( + nested=True + ) + atomic = serializers.BooleanField( + required=False, + default=True + ) + start = ObjectChangeSerializer( + nested=True, + required=False, + allow_null=True, + default=None + ) + end = ObjectChangeSerializer( + nested=True, + required=False, + allow_null=True, + default=None + ) diff --git a/netbox_branching/api/views.py b/netbox_branching/api/views.py index 8d115b3..b485392 100644 --- a/netbox_branching/api/views.py +++ b/netbox_branching/api/views.py @@ -1,6 +1,7 @@ from django.core.exceptions import PermissionDenied from django.http import HttpResponseBadRequest from drf_spectacular.utils import extend_schema +from rest_framework import status from rest_framework.decorators import action from rest_framework.mixins import ListModelMixin, RetrieveModelMixin from rest_framework.response import Response @@ -10,7 +11,7 @@ from core.api.serializers import JobSerializer from netbox.api.viewsets import BaseViewSet, NetBoxReadOnlyModelViewSet from netbox_branching import filtersets -from netbox_branching.jobs import MergeBranchJob, RevertBranchJob, SyncBranchJob +from netbox_branching.jobs import MergeBranchJob, PullBranchJob, RevertBranchJob, SyncBranchJob from netbox_branching.models import Branch, BranchEvent, ChangeDiff from . import serializers @@ -54,6 +55,43 @@ def sync(self, request, pk): return Response(JobSerializer(job, context={'request': request}).data) + @extend_schema( + methods=['post'], + request=serializers.BranchPullSerializer(), + responses={200: JobSerializer()}, + ) + @action(detail=True, methods=['post']) + def pull(self, request, pk): + """ + Enqueue a background job to pull changes from one Branch into another. + """ + if not request.user.has_perm('netbox_branching.pull_branch'): + raise PermissionDenied("This user does not have permission to pull branches.") + + branch = self.get_object() + if not branch.ready: + return HttpResponseBadRequest("Branch is not ready to apply changes.") + + serializer = serializers.BranchPullSerializer(data=request.data) + if not serializer.is_valid(): + return Response( + serializer.errors, + status=status.HTTP_400_BAD_REQUEST + ) + + # Enqueue a background job + job = PullBranchJob.enqueue( + instance=branch, + user=request.user, + source=serializer.validated_data['source'], + atomic=serializer.validated_data['atomic'], + start=serializer.validated_data['start'], + end=serializer.validated_data['end'], + commit=serializer.validated_data['commit'] + ) + + return Response(JobSerializer(job, context={'request': request}).data) + @extend_schema( methods=['post'], request=serializers.CommitSerializer(), diff --git a/netbox_branching/choices.py b/netbox_branching/choices.py index 90c6f46..e3745d0 100644 --- a/netbox_branching/choices.py +++ b/netbox_branching/choices.py @@ -43,6 +43,7 @@ class BranchStatusChoices(ChoiceSet): class BranchEventTypeChoices(ChoiceSet): PROVISIONED = 'provisioned' SYNCED = 'synced' + PULLED = 'pulled' MERGED = 'merged' REVERTED = 'reverted' ARCHIVED = 'archived' @@ -50,6 +51,7 @@ class BranchEventTypeChoices(ChoiceSet): CHOICES = ( (PROVISIONED, _('Provisioned'), 'green'), (SYNCED, _('Synced'), 'cyan'), + (PULLED, _('Pulled'), 'blue'), (MERGED, _('Merged'), 'blue'), (REVERTED, _('Reverted'), 'orange'), (ARCHIVED, _('Archived'), 'gray'), diff --git a/netbox_branching/constants.py b/netbox_branching/constants.py index 5b3fc7f..7649801 100644 --- a/netbox_branching/constants.py +++ b/netbox_branching/constants.py @@ -10,6 +10,7 @@ # Branch actions BRANCH_ACTIONS = ( 'sync', + 'pull', 'merge', 'revert', 'archive', diff --git a/netbox_branching/forms/misc.py b/netbox_branching/forms/misc.py index 1bbc590..0ddb4f4 100644 --- a/netbox_branching/forms/misc.py +++ b/netbox_branching/forms/misc.py @@ -1,10 +1,13 @@ from django import forms from django.utils.translation import gettext_lazy as _ -from netbox_branching.models import ChangeDiff +from utilities.forms.utils import get_field_value +from utilities.forms.widgets import HTMXSelect +from netbox_branching.models import Branch, ChangeDiff, ObjectChange __all__ = ( 'BranchActionForm', + 'BranchPullForm', 'ConfirmationForm', ) @@ -12,7 +15,8 @@ class BranchActionForm(forms.Form): pk = forms.ModelMultipleChoiceField( queryset=ChangeDiff.objects.all(), - required=False + required=False, + widget=forms.HiddenInput() ) commit = forms.BooleanField( required=False, @@ -42,6 +46,48 @@ def clean(self): return self.cleaned_data +class BranchPullForm(BranchActionForm): + source = forms.ModelChoiceField( + queryset=Branch.objects.all(), + widget=HTMXSelect( + attrs={ + 'hx-target': 'body' + } + ) + ) + atomic = forms.BooleanField( + label=_('Atomic'), + required=False, + initial=True, + help_text=_('Complete only if all changes from the source branch are applied successfully.') + ) + # TODO: Populate choices for start & end fields dynamically + start = forms.ModelChoiceField( + queryset=ObjectChange.objects.none(), + required=False + ) + end = forms.ModelChoiceField( + queryset=ObjectChange.objects.none(), + required=False + ) + + field_order = ('source', 'atomic', 'start', 'end', 'commit') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.fields['source'].queryset = Branch.objects.exclude(pk=self.branch.pk) + + if source_id := get_field_value(self, 'source'): + try: + source = Branch.objects.get(pk=source_id) + unpulled_changes = self.branch.get_unpulled_changes(source) + self.fields['start'].queryset = unpulled_changes + self.fields['end'].queryset = unpulled_changes + except Branch.DoesNotExist: + pass + + class ConfirmationForm(forms.Form): confirm = forms.BooleanField( required=True, diff --git a/netbox_branching/forms/model_forms.py b/netbox_branching/forms/model_forms.py index 69fab95..cd198b5 100644 --- a/netbox_branching/forms/model_forms.py +++ b/netbox_branching/forms/model_forms.py @@ -1,7 +1,9 @@ -from netbox_branching.models import Branch +from django import forms +from django.utils.translation import gettext_lazy as _ from netbox.forms import NetBoxModelForm -from utilities.forms.fields import CommentField +from netbox_branching.models import Branch +from utilities.forms.fields import CommentField, DynamicModelChoiceField from utilities.forms.rendering import FieldSet __all__ = ( @@ -11,10 +13,29 @@ class BranchForm(NetBoxModelForm): fieldsets = ( - FieldSet('name', 'description', 'tags'), + FieldSet('name', 'description', 'clone_from', 'atomic', 'tags'), + ) + clone_from = DynamicModelChoiceField( + label=_('Clone from'), + queryset=Branch.objects.all(), + required=False + ) + atomic = forms.BooleanField( + label=_('Atomic'), + required=False, + initial=True, + help_text=_('Clone only if all changes from the source branch are applied successfully.') ) comments = CommentField() class Meta: model = Branch - fields = ('name', 'description', 'comments', 'tags') + fields = ('name', 'description', 'clone_from', 'atomic', 'comments', 'tags') + + def save(self, *args, **kwargs): + + if clone_from := self.cleaned_data.get('clone_from'): + self.instance._clone_source = clone_from + self.instance._clone_atomic = self.cleaned_data['atomic'] + + return super().save(*args, **kwargs) diff --git a/netbox_branching/jobs.py b/netbox_branching/jobs.py index 13f0199..2ea1a0f 100644 --- a/netbox_branching/jobs.py +++ b/netbox_branching/jobs.py @@ -10,6 +10,7 @@ __all__ = ( 'MergeBranchJob', 'ProvisionBranchJob', + 'PullBranchJob', 'RevertBranchJob', 'SyncBranchJob', ) @@ -38,7 +39,7 @@ def run(self, *args, **kwargs): logger.setLevel(logging.DEBUG) logger.addHandler(ListHandler(queue=get_job_log(self.job))) - # Provision the Branch + # Provision the Branch by copying the main schema branch = self.job.object branch.provision(user=self.job.user) @@ -112,6 +113,27 @@ def run(self, commit=True, *args, **kwargs): logger.info("Dry run completed; rolling back changes") +class PullBranchJob(JobRunner): + """ + Pull changes from one Branch into another. + """ + class Meta: + name = 'Pull branch' + + def run(self, **kwargs): + # Initialize logging + logger = logging.getLogger('netbox_branching.branch.pull') + logger.setLevel(logging.DEBUG) + logger.addHandler(ListHandler(queue=get_job_log(self.job))) + + # Pull changes from the source Branch + try: + branch = self.job.object + branch.pull(user=self.job.user, **kwargs) + except AbortTransaction: + logger.info("Dry run completed; rolling back changes") + + class RevertBranchJob(JobRunner): """ Revert changes from a merged Branch. diff --git a/netbox_branching/migrations/0003_branchevent_related_branch.py b/netbox_branching/migrations/0003_branchevent_related_branch.py new file mode 100644 index 0000000..7b158e5 --- /dev/null +++ b/netbox_branching/migrations/0003_branchevent_related_branch.py @@ -0,0 +1,17 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_branching', '0002_branch_schema_id_unique'), + ] + + operations = [ + migrations.AddField( + model_name='branchevent', + name='related_branch', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='netbox_branching.branch'), + ), + ] diff --git a/netbox_branching/models/branches.py b/netbox_branching/models/branches.py index b711cbb..95f3781 100644 --- a/netbox_branching/models/branches.py +++ b/netbox_branching/models/branches.py @@ -1,6 +1,7 @@ import logging import random import string +from contextlib import nullcontext from datetime import timedelta from functools import cached_property, partial @@ -13,7 +14,6 @@ from django.test import RequestFactory from django.urls import reverse from django.utils import timezone -from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from core.models import ObjectChange as ObjectChange_ @@ -86,6 +86,7 @@ class Branch(JobsMixin, PrimaryModel): _preaction_validators = { 'sync': set(), + 'pull': set(), 'merge': set(), 'revert': set(), 'archive': set(), @@ -112,6 +113,16 @@ def get_absolute_url(self): def get_status_color(self): return BranchStatusChoices.colors.get(self.status) + def clone(self): + """ + Override CloningMixin's clone() method to nullify active branch and populate clone_from field. + """ + return { + '_branch': '', + 'clone_from': self.pk, + **super().clone(), + } + @cached_property def is_active(self): return self == active_branch.get() @@ -167,7 +178,7 @@ def save(self, provision=True, *args, **kwargs): provision: If True, automatically enqueue a background Job to provision the Branch. (Set this to False if you will call provision() on the instance manually.) """ - from netbox_branching.jobs import ProvisionBranchJob + from netbox_branching.jobs import ProvisionBranchJob, PullBranchJob _provision = provision and self.pk is None @@ -181,6 +192,16 @@ def save(self, provision=True, *args, **kwargs): user=request.user if request else None ) + # If cloning from an existing Branch, also enqueue a PullBranchJob + if clone_source := getattr(self, '_clone_source', None): + PullBranchJob.enqueue( + instance=self, + user=request.user, + source=clone_source, + atomic=getattr(self, '_clone_atomic', True), + commit=True + ) + def delete(self, *args, **kwargs): if active_branch.get() == self: raise AbortRequest(_("The active branch cannot be deleted.")) @@ -246,6 +267,28 @@ def get_merged_changes(self): application__branch=self ) + def get_unpulled_changes(self, source, start=None, end=None): + """ + Return a queryset of all ObjectChange records from the source Branch which have yet to be replayed onto + this Branch. + """ + if source.status not in BranchStatusChoices.WORKING: + return ObjectChange.objects.none() + + changes = ObjectChange.objects.using(source.connection_name).order_by('time') + + # Filter by starting change (if specified), or the time of the most recent pull event. + if start: + changes = changes.filter(pk__gte=start.pk) + elif last_pull := self.events.filter(related_branch=source, type=BranchEventTypeChoices.PULLED).first(): + changes = changes.filter(time__gt=last_pull.time) + + # Filter by end change (if specified) + if end: + changes = changes.filter(pk__lte=end.pk) + + return changes + def get_event_history(self): history = [] last_time = timezone.now() @@ -300,6 +343,13 @@ def can_sync(self): """ return self._can_do_action('sync') + @cached_property + def can_pull(self): + """ + Indicates whether changes can be pulled in from another Branch. + """ + return self._can_do_action('pull') + @cached_property def can_merge(self): """ @@ -386,6 +436,62 @@ def sync(self, user, commit=True): sync.alters_data = True + def pull(self, source, user, atomic=True, start=None, end=None, commit=True): + """ + Replicate all unpulled changes from the source branch into this one. + """ + logger = logging.getLogger('netbox_branching.branch.pull') + logger.info(f'Pulling changes from branch {source} into {self.name}') + + if not self.ready: + raise Exception(f"Branch {self} is not ready for changes.") + if not source.ready: + raise Exception(f"Changes cannot be pulled from branch {source} at this time.") + if commit and not self.can_pull: + raise Exception(f"Pulling changes to this branch is not permitted.") + + # Emit pre-pull signal + pre_pull.send(sender=self.__class__, branch=self, user=user) + + # Retrieve staged changes before we update the Branch's status + if changes := self.get_unpulled_changes(source, start=start, end=end): + logger.info(f"Found {len(changes)} changes to pull") + else: + logger.info(f"No changes found; aborting.") + return + + # Create a dummy request for the event_tracking() context manager + request = RequestFactory().get(reverse('home')) + + try: + use_atomic = atomic or not commit + with transaction.atomic(using=self.connection_name) if use_atomic else nullcontext(): + # Apply each change from the Branch + for change in changes: + with event_tracking(request): + request.id = change.request_id + request.user = change.user + change.apply(using=self.connection_name, logger=logger) + if not commit: + raise AbortTransaction() + + except Exception as e: + if err_message := str(e): + logger.error(err_message) + if atomic: + raise e + + # Record a branch event for the merge + logger.debug(f"Recording branch event: {BranchEventTypeChoices.PULLED}") + BranchEvent.objects.create(branch=self, related_branch=source, user=user, type=BranchEventTypeChoices.PULLED) + + # Emit post-pull signal + post_pull.send(sender=self.__class__, branch=self, user=user) + + logger.info('Pull completed') + + pull.alters_data = True + def merge(self, user, commit=True): """ Apply all changes in the Branch to the main schema by replaying them in @@ -681,6 +787,13 @@ class BranchEvent(models.Model): on_delete=models.CASCADE, related_name='events' ) + related_branch = models.ForeignKey( + to='netbox_branching.branch', + on_delete=models.CASCADE, + related_name='+', + blank=True, + null=True + ) user = models.ForeignKey( to=get_user_model(), on_delete=models.SET_NULL, diff --git a/netbox_branching/models/changes.py b/netbox_branching/models/changes.py index 1890823..74abfb9 100644 --- a/netbox_branching/models/changes.py +++ b/netbox_branching/models/changes.py @@ -27,6 +27,14 @@ class ObjectChange(ObjectChange_): class Meta: proxy = True + def __str__(self): + return '{}: {} {} by {}'.format( + self.pk, + self.object_repr, + self.get_action_display().lower(), + self.user_name + ) + def apply(self, using=DEFAULT_DB_ALIAS, logger=None): """ Apply the change using the specified database connection. diff --git a/netbox_branching/signals.py b/netbox_branching/signals.py index 063f6c4..fe34d6e 100644 --- a/netbox_branching/signals.py +++ b/netbox_branching/signals.py @@ -4,11 +4,13 @@ 'post_deprovision', 'post_merge', 'post_provision', + 'post_pull', 'post_revert', 'post_sync', 'pre_deprovision', 'pre_merge', 'pre_provision', + 'pre_pull', 'pre_revert', 'pre_sync', ) @@ -17,6 +19,7 @@ pre_provision = Signal() pre_deprovision = Signal() pre_sync = Signal() +pre_pull = Signal() pre_merge = Signal() pre_revert = Signal() @@ -24,5 +27,6 @@ post_provision = Signal() post_deprovision = Signal() post_sync = Signal() +post_pull = Signal() post_merge = Signal() post_revert = Signal() diff --git a/netbox_branching/templates/netbox_branching/branch.html b/netbox_branching/templates/netbox_branching/branch.html index 08705d8..8a4bd0c 100644 --- a/netbox_branching/templates/netbox_branching/branch.html +++ b/netbox_branching/templates/netbox_branching/branch.html @@ -24,6 +24,7 @@ {% endif %} {% if object.ready %} {% branch_sync_button object %} + {% branch_pull_button object %} {% branch_merge_button object %} {% endif %} {% if object.merged %} @@ -141,8 +142,14 @@
{% trans "Event History" %}
{% badge event.get_type_display bg_color=event.get_type_color %}
- {{ event.get_type_display }}{% if event.user %} by {{ event.user }}{% endif %} - {{ event.time|isodatetime }} + + {{ event.get_type_display }} + {% if event.related_branch %} {{ event.related_branch|linkify }}{% endif %} + + + {{ event.time|isodatetime }} + {% if event.user %}· {{ event.user }}{% endif %} +
{% else %} {# Change summary #} diff --git a/netbox_branching/templates/netbox_branching/branch_action.html b/netbox_branching/templates/netbox_branching/branch_action.html index 535872b..a0139f4 100644 --- a/netbox_branching/templates/netbox_branching/branch_action.html +++ b/netbox_branching/templates/netbox_branching/branch_action.html @@ -49,7 +49,9 @@ {% endif %}
- {% render_field form.commit %} +
+ {% render_form form %} +
{% trans "Cancel" %}