Closes #12988: Introduce custom field choice sets (#13195)

* Initial work on custom field choice sets

* Rename choices to extra_choices (prep for #12194)

* Remove CustomField.choices

* Add & update tests

* Clean up table columns

* Add order_alphanetically boolean for choice sets

* Introduce ArrayColumn for choice lists

* Show dependent custom fields on choice set view

* Update custom fields documentation

* Introduce ArrayWidget for more convenient editing of choices

* Incorporate PR feedback

* Misc cleanup

Author: jeremystretch

docs/customization/custom-fields.md

@@ -60,7 +60,7 @@ NetBox supports limited custom validation for custom field values. Following are
 
 ### Custom Selection Fields
 
-Each custom selection field must have at least two choices. These are specified as a comma-separated list. Choices appear in forms in the order they are listed. Note that choice values are saved exactly as they appear, so it's best to avoid superfluous punctuation or symbols where possible.
+Each custom selection field must designate a [choice set](../models/extras/customfieldchoiceset.md) containing at least two choices. These are specified as a comma-separated list.
 
 If a default value is specified for a selection field, it must exactly match one of the provided choices. The value of a multiple selection field will always return a list, even if only one value is selected.
 

docs/models/extras/customfield.md

@@ -79,9 +79,9 @@ Controls how and whether the custom field is displayed within the NetBox user in
 
 The default value to populate for the custom field when creating new objects (optional). This value must be expressed as JSON. If this is a choice or multi-choice field, this must be one of the available choices.
 
-### Choices
+### Choice Set
 
-For choice and multi-choice custom fields only. A comma-delimited list of the available choices.
+For selection and multi-select custom fields only, this is the [set of choices](./customfieldchoiceset.md) which are valid for the field.
 
 ### Cloneable
 

docs/models/extras/customfieldchoiceset.md

@@ -0,0 +1,17 @@
+# Custom Field Choice Sets
+
+Single- and multi-selection [custom fields documentation](../../customization/custom-fields.md) must define a set of valid choices from which the user may choose when defining the field value. These choices are defined as sets that may be reused among multiple custom fields.
+
+## Fields
+
+### Name
+
+The human-friendly name of the choice set.
+
+### Extra Choices
+
+The list of valid choices, entered as a comma-separated list.
+
+### Order Alphabetically
+
+If enabled, the choices list will be automatically ordered alphabetically. If disabled, choices will appear in the order in which they were defined.

netbox/extras/api/nested_serializers.py

@@ -7,6 +7,7 @@
     'NestedBookmarkSerializer',
     'NestedConfigContextSerializer',
     'NestedConfigTemplateSerializer',
+    'NestedCustomFieldChoiceSetSerializer',
     'NestedCustomFieldSerializer',
     'NestedCustomLinkSerializer',
     'NestedExportTemplateSerializer',
@@ -34,6 +35,14 @@ class Meta:
         fields = ['id', 'url', 'display', 'name']
 
 
+class NestedCustomFieldChoiceSetSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='extras-api:customfieldchoiceset-detail')
+
+    class Meta:
+        model = models.CustomFieldChoiceSet
+        fields = ['id', 'url', 'display', 'name', 'choices_count']
+
+
 class NestedCustomLinkSerializer(WritableNestedSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='extras-api:customlink-detail')
 

netbox/extras/api/serializers.py

@@ -35,6 +35,7 @@
     'ConfigContextSerializer',
     'ConfigTemplateSerializer',
     'ContentTypeSerializer',
+    'CustomFieldChoiceSetSerializer',
     'CustomFieldSerializer',
     'CustomLinkSerializer',
     'DashboardSerializer',
@@ -94,14 +95,15 @@ class CustomFieldSerializer(ValidatedModelSerializer):
     )
     filter_logic = ChoiceField(choices=CustomFieldFilterLogicChoices, required=False)
     data_type = serializers.SerializerMethodField()
+    choice_set = NestedCustomFieldChoiceSetSerializer(required=False)
     ui_visibility = ChoiceField(choices=CustomFieldVisibilityChoices, required=False)
 
     class Meta:
         model = CustomField
         fields = [
             'id', 'url', 'display', 'content_types', 'type', 'object_type', 'data_type', 'name', 'label', 'group_name',
             'description', 'required', 'search_weight', 'filter_logic', 'ui_visibility', 'is_cloneable', 'default',
-            'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'created',
+            'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choice_set', 'created',
             'last_updated',
         ]
 
@@ -127,6 +129,17 @@ def get_data_type(self, obj):
         return 'string'
 
 
+class CustomFieldChoiceSetSerializer(ValidatedModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='extras-api:customfieldchoiceset-detail')
+
+    class Meta:
+        model = CustomFieldChoiceSet
+        fields = [
+            'id', 'url', 'display', 'name', 'description', 'extra_choices', 'order_alphabetically', 'choices_count',
+            'created', 'last_updated',
+        ]
+
+
 #
 # Custom links
 #

netbox/extras/api/urls.py

@@ -9,6 +9,7 @@
 
 router.register('webhooks', views.WebhookViewSet)
 router.register('custom-fields', views.CustomFieldViewSet)
+router.register('custom-field-choices', views.CustomFieldChoiceSetViewSet)
 router.register('custom-links', views.CustomLinkViewSet)
 router.register('export-templates', views.ExportTemplateViewSet)
 router.register('saved-filters', views.SavedFilterViewSet)

netbox/extras/api/views.py

@@ -1,6 +1,5 @@
 from django.contrib.contenttypes.models import ContentType
 from django.http import Http404
-from django.shortcuts import get_object_or_404
 from django_rq.queues import get_connection
 from rest_framework import status
 from rest_framework.decorators import action
@@ -55,11 +54,17 @@ class WebhookViewSet(NetBoxModelViewSet):
 
 class CustomFieldViewSet(NetBoxModelViewSet):
     metadata_class = ContentTypeMetadata
-    queryset = CustomField.objects.all()
+    queryset = CustomField.objects.select_related('choice_set')
     serializer_class = serializers.CustomFieldSerializer
     filterset_class = filtersets.CustomFieldFilterSet
 
 
+class CustomFieldChoiceSetViewSet(NetBoxModelViewSet):
+    queryset = CustomFieldChoiceSet.objects.all()
+    serializer_class = serializers.CustomFieldChoiceSetSerializer
+    filterset_class = filtersets.CustomFieldChoiceSetFilterSet
+
+
 #
 # Custom links
 #

netbox/extras/filtersets.py

@@ -20,6 +20,7 @@
     'ConfigRevisionFilterSet',
     'ConfigTemplateFilterSet',
     'ContentTypeFilterSet',
+    'CustomFieldChoiceSetFilterSet',
     'CustomFieldFilterSet',
     'CustomLinkFilterSet',
     'ExportTemplateFilterSet',
@@ -74,6 +75,14 @@ class CustomFieldFilterSet(BaseFilterSet):
         field_name='content_types__id'
     )
     content_types = ContentTypeFilter()
+    choice_set_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=CustomFieldChoiceSet.objects.all()
+    )
+    choice_set = django_filters.ModelMultipleChoiceFilter(
+        field_name='choice_set__name',
+        queryset=CustomFieldChoiceSet.objects.all(),
+        to_field_name='name'
+    )
 
     class Meta:
         model = CustomField
@@ -93,6 +102,35 @@ def search(self, queryset, name, value):
         )
 
 
+class CustomFieldChoiceSetFilterSet(BaseFilterSet):
+    q = django_filters.CharFilter(
+        method='search',
+        label=_('Search'),
+    )
+    choice = MultiValueCharFilter(
+        method='filter_by_choice'
+    )
+
+    class Meta:
+        model = CustomFieldChoiceSet
+        fields = [
+            'id', 'name', 'description', 'order_alphabetically',
+        ]
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(name__icontains=value) |
+            Q(description__icontains=value) |
+            Q(extra_choices__contains=value)
+        )
+
+    def filter_by_choice(self, queryset, name, value):
+        # TODO: Support case-insensitive matching
+        return queryset.filter(extra_choices__overlap=value)
+
+
 class CustomLinkFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',

netbox/extras/forms/bulk_edit.py

@@ -4,13 +4,14 @@
 from extras.choices import *
 from extras.models import *
 from utilities.forms import BulkEditForm, add_blank_choice
-from utilities.forms.fields import ColorField
+from utilities.forms.fields import ColorField, DynamicModelChoiceField
 from utilities.forms.widgets import BulkEditNullBooleanSelect
 
 __all__ = (
     'ConfigContextBulkEditForm',
     'ConfigTemplateBulkEditForm',
     'CustomFieldBulkEditForm',
+    'CustomFieldChoiceSetBulkEditForm',
     'CustomLinkBulkEditForm',
     'ExportTemplateBulkEditForm',
     'JournalEntryBulkEditForm',
@@ -38,6 +39,10 @@ class CustomFieldBulkEditForm(BulkEditForm):
     weight = forms.IntegerField(
         required=False
     )
+    choice_set = DynamicModelChoiceField(
+        queryset=CustomFieldChoiceSet.objects.all(),
+        required=False
+    )
     ui_visibility = forms.ChoiceField(
         label=_("UI visibility"),
         choices=add_blank_choice(CustomFieldVisibilityChoices),
@@ -49,7 +54,23 @@ class CustomFieldBulkEditForm(BulkEditForm):
         widget=BulkEditNullBooleanSelect()
     )
 
-    nullable_fields = ('group_name', 'description',)
+    nullable_fields = ('group_name', 'description', 'choice_set')
+
+
+class CustomFieldChoiceSetBulkEditForm(BulkEditForm):
+    pk = forms.ModelMultipleChoiceField(
+        queryset=CustomFieldChoiceSet.objects.all(),
+        widget=forms.MultipleHiddenInput
+    )
+    description = forms.CharField(
+        required=False
+    )
+    order_alphabetically = forms.NullBooleanField(
+        required=False,
+        widget=BulkEditNullBooleanSelect()
+    )
+
+    nullable_fields = ('description',)
 
 
 class CustomLinkBulkEditForm(BulkEditForm):

netbox/extras/forms/bulk_import.py

@@ -9,10 +9,13 @@
 from extras.utils import FeatureQuery
 from netbox.forms import NetBoxModelImportForm
 from utilities.forms import CSVModelForm
-from utilities.forms.fields import CSVChoiceField, CSVContentTypeField, CSVMultipleContentTypeField, SlugField
+from utilities.forms.fields import (
+    CSVChoiceField, CSVContentTypeField, CSVModelChoiceField, CSVMultipleContentTypeField, SlugField,
+)
 
 __all__ = (
     'ConfigTemplateImportForm',
+    'CustomFieldChoiceSetImportForm',
     'CustomFieldImportForm',
     'CustomLinkImportForm',
     'ExportTemplateImportForm',
@@ -39,10 +42,11 @@ class CustomFieldImportForm(CSVModelForm):
         required=False,
         help_text=_("Object type (for object or multi-object fields)")
     )
-    choices = SimpleArrayField(
-        base_field=forms.CharField(),
+    choice_set = CSVModelChoiceField(
+        queryset=CustomFieldChoiceSet.objects.all(),
+        to_field_name='name',
         required=False,
-        help_text=_('Comma-separated list of field choices')
+        help_text=_('Choice set (for selection fields)')
     )
     ui_visibility = CSVChoiceField(
         choices=CustomFieldVisibilityChoices,
@@ -53,8 +57,22 @@ class Meta:
         model = CustomField
         fields = (
             'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description',
-            'search_weight', 'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum',
-            'validation_regex', 'ui_visibility', 'is_cloneable',
+            'search_weight', 'filter_logic', 'default', 'choice_set', 'weight', 'validation_minimum',
+            'validation_maximum', 'validation_regex', 'ui_visibility', 'is_cloneable',
+        )
+
+
+class CustomFieldChoiceSetImportForm(CSVModelForm):
+    extra_choices = SimpleArrayField(
+        base_field=forms.CharField(),
+        required=False,
+        help_text=_('Comma-separated list of field choices')
+    )
+
+    class Meta:
+        model = CustomFieldChoiceSet
+        fields = (
+            'name', 'description', 'extra_choices', 'order_alphabetically',
         )
 
 

netbox/extras/forms/filtersets.py

@@ -20,6 +20,7 @@
     'ConfigContextFilterForm',
     'ConfigRevisionFilterForm',
     'ConfigTemplateFilterForm',
+    'CustomFieldChoiceSetFilterForm',
     'CustomFieldFilterForm',
     'CustomLinkFilterForm',
     'ExportTemplateFilterForm',
@@ -37,7 +38,8 @@ class CustomFieldFilterForm(SavedFiltersMixin, FilterForm):
     fieldsets = (
         (None, ('q', 'filter_id')),
         ('Attributes', (
-            'type', 'content_type_id', 'group_name', 'weight', 'required', 'ui_visibility', 'is_cloneable',
+            'type', 'content_type_id', 'group_name', 'weight', 'required', 'choice_set_id', 'ui_visibility',
+            'is_cloneable',
         )),
     )
     content_type_id = ContentTypeMultipleChoiceField(
@@ -62,6 +64,11 @@ class CustomFieldFilterForm(SavedFiltersMixin, FilterForm):
             choices=BOOLEAN_WITH_BLANK_CHOICES
         )
     )
+    choice_set_id = DynamicModelMultipleChoiceField(
+        queryset=CustomFieldChoiceSet.objects.all(),
+        required=False,
+        label=_('Choice set')
+    )
     ui_visibility = forms.ChoiceField(
         choices=add_blank_choice(CustomFieldVisibilityChoices),
         required=False,
@@ -75,10 +82,19 @@ class CustomFieldFilterForm(SavedFiltersMixin, FilterForm):
     )
 
 
+class CustomFieldChoiceSetFilterForm(SavedFiltersMixin, FilterForm):
+    fieldsets = (
+        (None, ('q', 'filter_id', 'choice')),
+    )
+    choice = forms.CharField(
+        required=False
+    )
+
+
 class CustomLinkFilterForm(SavedFiltersMixin, FilterForm):
     fieldsets = (
         (None, ('q', 'filter_id')),
-        ('Attributes', ('content_types', 'enabled', 'new_window', 'weight')),
+        (_('Attributes'), ('content_types', 'enabled', 'new_window', 'weight')),
     )
     content_types = ContentTypeMultipleChoiceField(
         queryset=ContentType.objects.filter(FeatureQuery('custom_links').get_query()),