feat(tofs): first implemetation + tplroot

Sammy · nb · commit d5262ea5dfb4 · 2019-06-04T08:57:21.000+11:00
First implementation of TOFS for:
nginx_config, passenger_config, servers_config and snippets
Introduced tplroot on modified files as well
Fixed GH link of nginx.conf file in docs/TOFS_pattern.rst
Fixed test for snippets name pillar
diff --git a/docs/TOFS_pattern.rst b/docs/TOFS_pattern.rst
@@ -39,7 +39,7 @@ Using SaltStack is a simple and effective way to implement configuration managem
 
 To avoid this situation we can use the `pillar mechanism <http://docs.saltstack.com/en/latest/topics/pillar/>`_, which is designed to provide controlled access to data from the minions based on some selection rules. As pillar data could be easily integrated in the `Jinja <http://docs.saltstack.com/en/latest/topics/tutorials/pillar.html>`_ templates, it is a good mechanism to store values to be used in the final rendering of state files and templates.
 
-There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas <https://github.com/saltstack-formulas>`_' repositories. `Some <https://github.com/saltstack-formulas/nginx-formula/pull/18>`_ `developments <https://github.com/saltstack-formulas/php-formula/pull/14>`_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja <https://github.com/spsoit/nginx-formula/blob/81de880fe0276dd9488ffa15bc78944c0fc2b919/nginx/files/nginx.conf>`_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer.
+There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas <https://github.com/saltstack-formulas>`_' repositories. `Some <https://github.com/saltstack-formulas/nginx-formula/pull/18>`_ `developments <https://github.com/saltstack-formulas/php-formula/pull/14>`_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja <https://github.com/saltstack-formulas/nginx-formula/blob/f74254c07e188bd448eaf1c5f9c802d78c4c005e/nginx/files/default/nginx.conf>`_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer.
 
 In opposition to the *put the code in file_roots and the data in pillars* approach, there is the *pillar as a store for a set of key-values* approach. A full-blown configuration file abstracted in pillar and jinja is complicated to develop, understand and maintain. I think a better and simpler approach is to keep a configuration file templated using just a basic (non-extensive but extensible) set of pillar values.
 
@@ -432,7 +432,7 @@ The list of ``source_files`` can be given:
 Resulting in:
 
 .. code-block:: sls
-   
+
          - source:
            - salt://ntp/files/theminion/etc/ntp.conf.jinja
            - salt://ntp/files/theminion/etc/ntp.conf_alt.jinja
diff --git a/nginx/config.sls b/nginx/config.sls
@@ -2,7 +2,9 @@
 #
 # Manages the main nginx server configuration file.
 
-{% from 'nginx/map.jinja' import nginx, sls_block with context %}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %}
+{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %}
 
 {% if nginx.install_from_source %}
 nginx_log_dir:
@@ -12,16 +14,18 @@ nginx_log_dir:
     - group: {{ nginx.server.config.user }}
 {% endif %}
 
-{% if 'source_path' in nginx.server.config %}
-{% set source_path = nginx.server.config.source_path %}
-{% else %}
-{% set source_path = 'salt://nginx/files/nginx.conf' %}
-{% endif %}
 nginx_config:
   file.managed:
     {{ sls_block(nginx.server.opts) }}
     - name: {{ nginx.lookup.conf_file }}
-    - source: {{ source_path }}
+    - source:
+{% if 'source_path' in nginx.server.config %}
+      - {{ nginx.server.config.source_path }}
+{% endif %}
+      {{ files_switch(['nginx.conf'],
+                      'nginx_config_file_managed'
+          )
+      }}
     - template: jinja
 {% if 'source_path' not in nginx.server.config %}
     - context:
diff --git a/nginx/files/default/nginx.conf b/nginx/files/default/nginx.conf
diff --git a/nginx/files/default/nginx.service b/nginx/files/default/nginx.service
diff --git a/nginx/files/default/server.conf b/nginx/files/default/server.conf
diff --git a/nginx/libtofs.jinja b/nginx/libtofs.jinja
@@ -0,0 +1,101 @@
+{%- macro files_switch(source_files,
+                       lookup=None,
+                       default_files_switch=['id', 'os_family'],
+                       indent_width=6,
+                       v1_path_prefix='') %}
+  {#-
+    Returns a valid value for the "source" parameter of a "file.managed"
+    state function. This makes easier the usage of the Template Override and
+    Files Switch (TOFS) pattern.
+
+    Params:
+      * source_files: ordered list of files to look for
+      * lookup: key under '<tplroot>:tofs:source_files' to override
+        list of source files
+      * default_files_switch: if there's no config (e.g. pillar)
+        '<tplroot>:tofs:files_switch' this is the ordered list of grains to
+        use as selector switch of the directories under
+        "<path_prefix>/files"
+      * indent_witdh: indentation of the result value to conform to YAML
+      * v1_path_prefix: (deprecated) only used for injecting a path prefix into
+        the source, to support older TOFS configs
+
+    Example (based on a `tplroot` of `xxx`):
+
+    If we have a state:
+
+      Deploy configuration:
+        file.managed:
+          - name: /etc/yyy/zzz.conf
+          - source: {{ files_switch(['/etc/yyy/zzz.conf', '/etc/yyy/zzz.conf.jinja'],
+                                    lookup='Deploy configuration'
+                    ) }}
+          - template: jinja
+
+    In a minion with id=theminion and os_family=RedHat, it's going to be
+    rendered as:
+
+      Deploy configuration:
+        file.managed:
+          - name: /etc/yyy/zzz.conf
+          - source:
+            - salt://xxx/files/theminion/etc/yyy/zzz.conf
+            - salt://xxx/files/theminion/etc/yyy/zzz.conf.jinja
+            - salt://xxx/files/RedHat/etc/yyy/zzz.conf
+            - salt://xxx/files/RedHat/etc/yyy/zzz.conf.jinja
+            - salt://xxx/files/default/etc/yyy/zzz.conf
+            - salt://xxx/files/default/etc/yyy/zzz.conf.jinja
+          - template: jinja
+  #}
+  {#- Get the `tplroot` from `tpldir` #}
+  {%- set tplroot = tpldir.split('/')[0] %}
+  {%- set path_prefix = salt['config.get'](tplroot ~ ':tofs:path_prefix', tplroot) %}
+  {%- set files_dir = salt['config.get'](tplroot ~ ':tofs:dirs:files', 'files') %}
+  {%- set files_switch_list = salt['config.get'](
+      tplroot ~ ':tofs:files_switch',
+      default_files_switch
+  ) %}
+  {#- Lookup source_files (v2), files (v1), or fallback to source_files parameter #}
+  {%- set src_files = salt['config.get'](
+      tplroot ~ ':tofs:source_files:' ~ lookup,
+      salt['config.get'](
+          tplroot ~ ':tofs:files:' ~ lookup,
+          source_files
+      )
+  ) %}
+  {#- Only add to [''] when supporting older TOFS implementations #}
+  {%- set path_prefix_exts = [''] %}
+  {%- if v1_path_prefix != '' %}
+    {%- do path_prefix_exts.append(v1_path_prefix) %}
+  {%- endif %}
+  {%- for path_prefix_ext in path_prefix_exts %}
+    {%- set path_prefix_inc_ext = path_prefix ~ path_prefix_ext %}
+    {#- For older TOFS implementation, use `files_switch` from the config #}
+    {#- Use the default, new method otherwise #}
+    {%- set fsl = salt['config.get'](
+        tplroot ~ path_prefix_ext|replace('/', ':') ~ ':files_switch',
+        files_switch_list
+    ) %}
+    {#- Append an empty value to evaluate as `default` in the loop below #}
+    {%- if '' not in fsl %}
+      {%- do fsl.append('') %}
+    {%- endif %}
+    {%- for fs in fsl %}
+      {%- for src_file in src_files %}
+        {%- if fs %}
+          {%- set fs_dir = salt['config.get'](fs, fs) %}
+        {%- else %}
+          {%- set fs_dir = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %}
+        {%- endif %}
+        {%- set url = [
+            '- salt:/',
+            path_prefix_inc_ext.strip('/'),
+            files_dir.strip('/'),
+            fs_dir.strip('/'),
+            src_file.strip('/'),
+        ] | select | join('/') %}
+{{ url | indent(indent_width, true) }}
+      {%- endfor %}
+    {%- endfor %}
+  {%- endfor %}
+{%- endmacro %}
diff --git a/nginx/passenger.sls b/nginx/passenger.sls
@@ -3,7 +3,9 @@
 # Manages installation of passenger from repo.
 # Requires install_from_phusionpassenger = True
 
-{% from 'nginx/map.jinja' import nginx, sls_block with context %}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %}
+{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %}
 
 {% if salt['grains.get']('os_family') in ['Debian', 'RedHat'] %}
 include:
@@ -27,7 +29,10 @@ passenger_config:
   file.managed:
     {{ sls_block(nginx.server.opts) }}
     - name: {{ nginx.lookup.passenger_config_file }}
-    - source: salt://nginx/files/nginx.conf
+    - source: {{ files_switch(['nginx.conf'],
+                              'passenger_config_file_managed'
+                 )
+              }}
     - template: jinja
     - context:
         config: {{ nginx.passenger|json() }}
diff --git a/nginx/servers.sls b/nginx/servers.sls
@@ -2,9 +2,10 @@
 #
 # Manages virtual hosts and their relationship to the nginx service.
 
-{% from 'nginx/map.jinja' import nginx, sls_block with context %}
-{% from 'nginx/servers_config.sls' import server_states with context %}
-{% from 'nginx/service.sls' import service_function with context %}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %}
+{%- from tplroot ~ '/servers_config.sls' import server_states with context %}
+{%- from tplroot ~ '/service.sls' import service_function with context %}
 
 {% macro file_requisites(states) %}
       {%- for state in states %}
diff --git a/nginx/servers_config.sls b/nginx/servers_config.sls
@@ -2,7 +2,11 @@
 #
 # Manages the configuration of virtual host files.
 
-{% from 'nginx/map.jinja' import nginx, sls_block with context %}
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %}
+{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %}
+
 {% set server_states = [] %}
 
 # Simple path concatenation.
@@ -100,17 +104,19 @@ nginx_server_available_dir:
     file.absent:
         - name: {{ server_curpath(server) }}
 {% else %}
-{% if settings.config != None and settings.enabled == True %}
-{% if 'source_path' in settings.config %}
-{% set source_path = settings.config.source_path %}
-{% else %}
-{% set source_path = 'salt://nginx/files/server.conf' %}
-{% endif %}
+{% if settings.enabled == True %}
 {{ conf_state_id }}:
   file.managed:
     {{ sls_block(nginx.servers.managed_opts) }}
     - name: {{ server_curpath(server) }}
-    - source: {{ source_path }}
+    - source:
+{%- if 'source_path' in settings.config %}
+      - {{ settings.config.source_path }}
+{%- endif %}
+      {{ files_switch([server, 'server.conf'],
+                      'server_conf_file_managed'
+         )
+      }}
     - makedirs: True
     - template: jinja
     - require_in:
@@ -138,7 +144,7 @@ nginx_server_available_dir:
 {% else %}
 {{ manage_status(server, settings.enabled, False) }}
 {% endif %}
-{% if settings.config != None and settings.enabled == True %}
+{% if settings.enabled == True %}
     - require:
       - file: {{ conf_state_id }}
 {% endif %}
diff --git a/nginx/service.sls b/nginx/service.sls
@@ -2,7 +2,10 @@
 #
 # Manages the nginx service.
 
-{% from 'nginx/map.jinja' import nginx, sls_block with context %}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %}
+{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %}
+
 {% set service_function = {True:'running', False:'dead'}.get(nginx.service.enable) %}
 
 include:
@@ -16,7 +19,10 @@ include:
 nginx_systemd_service_file:
   file.managed:
     - name: /lib/systemd/system/nginx.service
-    - source: salt://nginx/files/nginx.service
+    - source: {{ files_switch(['nginx.service'],
+                              'nginx_systemd_service_file'
+                 )
+              }}
 {% endif %}
 
 nginx_service:
diff --git a/nginx/snippets.sls b/nginx/snippets.sls
@@ -2,7 +2,9 @@
 #
 # Manages creation of snippets
 
-{% from 'nginx/map.jinja' import nginx, sls_block with context %}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %}
+{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %}
 
 nginx_snippets_dir:
   file.directory:
@@ -12,8 +14,11 @@ nginx_snippets_dir:
 {% for snippet, config in nginx.snippets.items() %}
 nginx_snippet_{{ snippet }}:
   file.managed:
-    - name: {{ nginx.lookup.snippets_dir }}/{{ snippet }}.conf
-    - source: salt://nginx/files/server.conf
+    - name: {{ nginx.lookup.snippets_dir ~ '/' ~ snippet ~ '.conf' }}
+    - source: {{ files_switch([ snippet, 'server.conf' ],
+                              'nginx_snippet_file_managed'
+                 )
+              }}
     - template: jinja
     - context:
         config: {{ config|json() }}
diff --git a/pillar.example b/pillar.example
@@ -60,6 +60,7 @@ nginx:
   ##--- --- - - - - - - -- - - - - -- - - --- -- - -- - - - -- - - - - -- - - - -- - - - -- - ##
   ## You can use snippets to define often repeated configuration once and include it later
   ## The letsencrypt example below is consumed by "- include: 'snippets/letsencrypt.conf'"
+  ## Files or Templates can be retrieved by TOFS with snippet name ( Fallback to server.conf )
   ##--- --- - - - - - - --  - - -- -- - - --- -- - -- - - - -- - - - - -- - - - -- - - - -- - ##
   snippets:
     letsencrypt:
@@ -85,20 +86,21 @@ nginx:
         - server: 127.0.0.1:19999
         - keepalive: 64
 
-
   server:
     opts: {} # this partially exposes file.managed parameters as they relate to the main nginx.conf file
 
     #-- - - - - -- - - -- - - - - -- - - -- - - - -- - - - - - -- - - - - - -- - - - - -- - - - - -- - - #
     # nginx.conf (main server) declarations
     # dictionaries map to blocks {} and lists cause the same declaration to repeat with different values
     # see also http://nginx.org/en/docs/example.html
+    # Nginx config file or template can be retrieved by TOFS ( Fallback to nginx.conf )
     #-- - - - - -- - - -- - - - - -- - - -- - - - -- - - - - - -- - - - - - -- - - - - -- - - - - -- - - #
     config:
       include: 'snippets/letsencrypt.conf'
-      source_path: salt://path_to_nginx_conf_file/nginx.conf # IMPORTANT: This option is mutually exclusive with the rest of the
-                                                        # options; if it is found other options (worker_processes: 4 and so
-                                                        # on) are not processed and just upload the file from source
+      source_path: salt://path_to_nginx_conf_file/nginx.conf # IMPORTANT: This option is mutually exclusive with TOFS and
+                                                             # the rest of the options; if it is found other options
+                                                             # (worker_processes: 4 and so on) are not processed
+                                                             # and just upload the file from source
       worker_processes: 4
       load_module: modules/ngx_http_lua_module.so  # pass as very first in configuration; otherwise nginx will fail to start
       #pid: /var/run/nginx.pid                     # Directory location must exist (i.e. it's  /run/nginx.pid on EL7)
@@ -218,7 +220,8 @@ nginx:
         available_dir: /etc/nginx/sites-available
         enabled_dir: /etc/nginx/sites-enabled
         config:
-          source_path: salt://path-to-site-file/mysite2
+          source_path: salt://path-to-site-file/mysite2 # IMPORTANT: This field is mutually exclusive with TOFS
+                                                        # and other config options, it just uploads the specified file
 
       # Below configuration becomes handy if you want to create custom configuration files
       # for example if you want to create /usr/local/etc/nginx/http_options.conf with
@@ -277,7 +280,39 @@ nginx:
   # Passenger configuration
   # Default passenger configuration is provided, and will be deployed in
   # /etc/nginx/conf.d/passenger.conf
+  # Passenger conf can be retrieved by TOFS ( Fallback to nginx.conf )
   passenger:
     passenger_root: /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini
     passenger_ruby: /usr/bin/ruby
     passenger_instance_registry_dir: /var/run/passenger-instreg
+
+  tofs:
+    # The files_switch key serves as a selector for alternative
+    # directories under the formula files directory. See TOFS pattern
+    # doc for more info.
+    # Note: Any value not evaluated by `config.get` will be used literally.
+    # This can be used to set custom paths, as many levels deep as required.
+    # files_switch:
+    #   - any/path/can/be/used/here
+    #   - id
+    #   - role
+    #   - osfinger
+    #   - os
+    #   - os_family
+    # All aspects of path/file resolution are customisable using the options below.
+    # This is unnecessary in most cases; there are sensible defaults.
+    # path_prefix: template_alt
+    # dirs:
+    #   files: files_alt
+    #   default: default_alt
+    source_files:
+      nginx_config_file_managed:
+        - alt_nginx.conf
+      passenger_config_file_managed:
+        - alt_nginx.conf
+      server_conf_file_managed:
+        - alt_server.conf
+      nginx_systemd_service_file:
+        - alt_nginx.service
+      nginx_snippet_file_managed:
+        - alt_server.conf
