Change to dynamically construct API set from Flask

Note that the unit test remains as it was, as a check that we're
generating the proper set; this will have to be maintained as our
API expands, but that seems like a worthwhile validation.

Author: dbutenhof

lib/pbench/server/api/resources/endpoint_configure.py

@@ -1,4 +1,5 @@
 import re
+from flask.globals import current_app
 
 from flask_restful import Resource, abort
 from flask import request, jsonify
@@ -14,6 +15,10 @@ class EndpointConfig(Resource):
     config file.
     """
 
+    forward_pattern = re.compile(r";\s*host\s*=\s*(?P<host>[^;\s]*)")
+    x_forward_pattern = re.compile(r"^\s*(?P<host>[^;\s,]*)")
+    param_template = re.compile(r"<[\w_\d]+:[\d_\w]+>")
+
     def __init__(self, config, logger):
         """
         __init__ Construct the API resource
@@ -22,21 +27,19 @@ def __init__(self, config, logger):
             config (PbenchServerConfig): server config values
             logger (Logger): message logging
 
-        Report the server configuration to a web client; web access can be
-        redirected through an external reverse proxy like NGINX by setting
-        the "proxy_host" configuration setting in the "pbench-server"
-        section of the pbench-server.cfg file. By default, the Pbench
-        server uses a local Apache reverse proxy routing through the HTTP
-        port (80), but this can be changed to any host name and port. All
-        server endpoints will be reported with respect to that address.
+        Report the server configuration to a web client. By default, the Pbench
+        server ansible script sets up a local Apache reverse proxy routing
+        through the HTTP port (80); an external reverse-proxy can be configured
+        without the knowledge of the server, and this API will use reverse-proxy
+        Forwarded or X-Forwarded-Host HTTP headers to discover the proxy
+        configuration. All server endpoints will be reported with respect to that
+        address.
         """
         self.logger = logger
         self.host = config.get("pbench-server", "host")
         self.uri_prefix = config.rest_uri
         self.prefix = get_index_prefix(config)
         self.commit_id = config.COMMIT_ID
-        self.forward_pattern = re.compile(r";\s*host\s*=\s*(?P<host>[^;\s]*)")
-        self.x_forward_pattern = re.compile(r"^\s*(?P<host>[^;\s,]*)")
 
     def get(self):
         """
@@ -56,25 +59,17 @@ def get(self):
                 schema, this is "v6.run-data."
             run_toc_index: The Elasticsearch V7 index for run TOC data. In
                 the current ES schema, this is "v6.run-toc."
-        api:    A list of the server APIs supported; we give a name, which
+        api:    A dict of the server APIs supported; we give a name, which
                 identifies the service, and the full URI relative to the
-                configured host name and port (local or remote reverse proxy)
-            results: Direct access to Pbench data sets for the dashboard; this
-                is currently direct HTTP access through Apache public_html, and
-                not an "API" as such; we will need to adopt this as a real
-                server API when Pbench moves from local filesystem to S3 storage
-                schemas
-            elasticsearch: The Pbench pass-through URI for the Elasticsearch
-                cluster. This will eventually be superceded by the native
-                Pbench APIs, though it might remain accessible with special
-                user/group privileges to support special cases
-            graphql: The GraphQL frontend on postgreSQL currently used by the
-                dashboard user mocks. This will be superceded and deprecated
-                by native Pbench user management APIs
-            queryControllers: Return information about the run documents that
-                were created within a specified range of months.
-            queryMonthIndices: Return the YYYY-mm date strings for all ES
-                vx.run-data.* indices, in descending order
+                configured host name and port (local or remote reverse proxy).
+
+                This is dynamically generated by processing the Flask URI
+                rules; refer to api/__init__.py for the code which creates
+                those mappings, or test_endpoint_configure.py for code that
+                validates the current set (and must be updated when the API
+                set changes). We supplement the Flask API list with the
+                "results" API, which is currently just an Apache public_html
+                file mapping but is referenced by the dashboard code.
 
         TODO: We need an internal mechanism to track the active versions of the
         various Elasticsearch template documents. We're hardcoding them here and
@@ -86,31 +81,70 @@ def get(self):
         implementations. The entire "indices" section can be removed once that is
         resolved.
         """
-        self.logger.info("Received these headers: {!r}", request.headers)
-        header = request.headers.get("Forwarded")
+        self.logger.debug("Received these headers: {!r}", request.headers)
         origin = None
+        host_source = "configuration"
+        host_value = self.host
+        header = request.headers.get("Forwarded")
         if header:
             m = self.forward_pattern.search(header)
             if m:
                 origin = m.group("host")
-                self.logger.info("Forwarded: {}, ({})", header, origin)
+                host_source = "Forwarded"
+                host_value = header
         if not origin:
             header = request.headers.get("X-Forwarded-Host")
             if header:
                 m = self.x_forward_pattern.search(header)
                 if m:
                     origin = m.group("host")
-                    self.logger.info("X-Forwarded-Host: {}, ({})", header, origin)
+                    host_source = "X-Forwarded-Host"
+                    host_value = header
         if not origin:
             origin = self.host
         host = f"http://{origin}"
-        uri = urljoin(host, self.uri_prefix)
-        self.logger.info("'{}' : '{}' : '{}' : '{}'", self.host, origin, host, uri)
+        self.logger.info(
+            "Advertising endpoints relative to {}: {}", host_source, host_value
+        )
 
-        # Strip a trailing slash because the paths we'll add have an
-        # initial slash and we don't want two.
-        if uri.endswith("/"):
-            uri = uri.split(0, -1)
+        # We pre-load the APIs list with the "results" link, which isn't yet
+        # a Pbench server API. The dashboard static endpoint configuration
+        # included this, and it makes sense for consistency.
+        apis = {"results": urljoin(host, "/results")}
+
+        # Iterate through the Flask endpoints to add a description for each.
+        for rule in current_app.url_map.iter_rules():
+            url = str(rule.rule)
+
+            # Ignore anything that doesn't use our API prefix, because it's
+            # not in our API.
+            if url.startswith(self.uri_prefix):
+                # If the URI is parameterized with a Flask "<type:name>"
+                # template string, we don't want to report it, so we remove
+                # it from the URI. We derive an API name by converting the
+                # "/" characters in the URI to "_", after removing a trailing
+                # "/" that would have been left by removing a template... we
+                # don't remove the trailing "/" from the URI, which serves as
+                # an indication that the parameter is needed.
+                #
+                # E.g, "/api/v1/controllers/list" yields:
+                #     "controllers_list": "/api/v1/controllers/list"
+                #
+                # while "/api/v1/users/<string:username>" yields:
+                #     "users": "/api/v1/users/"
+                #
+                # TODO: This won't work right with embedded template strings,
+                # which we're not currently using anywhere; but it'll require
+                # adjustment later if we add any. (E.g., something like
+                # "/api/v1/foo/<string:name>/detail/<string:param>")
+                m = re.search(self.param_template, url)
+                if m:
+                    url = re.sub(self.param_template, "", url)
+                path = url[len(self.uri_prefix) + 1 :]
+                if path.endswith("/"):
+                    path = path[:-1]
+                path = path.replace("/", "_")
+                apis[path] = urljoin(host, url)
 
         try:
             endpoints = {
@@ -121,20 +155,7 @@ def get(self):
                     "result_index": f"{self.prefix}.v5.result-data-sample.",
                     "result_data_index": f"{self.prefix}.v5.result-data.",
                 },
-                "api": {
-                    "results": f"{host}/results",
-                    "elasticsearch": f"{uri}/elasticsearch",
-                    "endpoints": f"{uri}/endpoints",
-                    "graphql": f"{uri}/graphql",
-                    "controller_list": f"{uri}/controllers/list",
-                    "controller_months": f"{uri}/controllers/months",
-                    "dataset_list": f"{uri}/datasets/list",
-                    "dataset": f"{uri}/datasets",
-                    "register": f"{uri}/register",
-                    "login": f"{uri}/login",
-                    "logout": f"{uri}/logout",
-                    "user": f"{uri}/user/<string:username>",
-                },
+                "api": apis,
             }
             response = jsonify(endpoints)
         except Exception:

lib/pbench/test/unit/server/test_endpoint_configure.py

@@ -53,14 +53,14 @@ def check_config(self, client, server_config, host, my_headers={}):
                 "elasticsearch": f"{uri}/elasticsearch",
                 "endpoints": f"{uri}/endpoints",
                 "graphql": f"{uri}/graphql",
-                "controller_list": f"{uri}/controllers/list",
-                "controller_months": f"{uri}/controllers/months",
-                "dataset_list": f"{uri}/datasets/list",
-                "dataset": f"{uri}/datasets",
+                "controllers_list": f"{uri}/controllers/list",
+                "controllers_months": f"{uri}/controllers/months",
                 "register": f"{uri}/register",
                 "login": f"{uri}/login",
                 "logout": f"{uri}/logout",
-                "user": f"{uri}/user/<string:username>",
+                "user": f"{uri}/user/",
+                "host_info": f"{uri}/host_info",
+                "upload_ctrl": f"{uri}/upload/ctrl/",
             },
         }