Add support for moving URLs and setting canonical URL correctly (#4115)

Author: mheap

app/_plugins/generators/canonical_url_generator.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
+require 'yaml'
 module CanonicalUrl
   class Generator < Jekyll::Generator # rubocop:disable Metrics/ClassLength
     priority :low
     def generate(site)
+      # We need to keep track of renamed pages in some cases
+      # Usually when we rework the IA between major releases but want to keep
+      # the SEO juice for a given page
+      @moved_pages = YAML.load_file("#{__dir__}/../../moved_urls.yml")
+
       # Generate the all_pages entres for the Plugin Hub
       all_pages = generate_plugin_hub(site)
 
@@ -108,6 +114,11 @@ def set_canonical_and_noindex(site, all_pages) # rubocop:disable Metrics/AbcSize
         url = page.url.gsub(url_segments[1], 'VERSION')
         urls_to_check << url
 
+        # If a page has been renamed between versions, then we need to check
+        # for the new URL in later versions too
+        moved_url = resolve_moved_url(url)
+        urls_to_check << moved_url if moved_url
+
         # As before, legacy endpoints might match newer /gateway/ URLs so
         # we also need to check for the path under the /gateway/ docs too
         legacy_gateway_endpoints = ['/gateway-oss/', '/enterprise/']
@@ -117,11 +128,18 @@ def set_canonical_and_noindex(site, all_pages) # rubocop:disable Metrics/AbcSize
 
         # There will usually only be one URL to check, but gateway-oss
         # and enterprise URLs will contain two here, so we have to loop
+        latest_version = to_version('0.0.x')
+        canonical_url = nil
+
         urls_to_check.each do |u|
           # Otherwise look up the URL and link to the latest version
           matching_url = all_pages[u]
-          page.data['canonical_url'] = matching_url['url'] if matching_url
+          next unless matching_url && matching_url['version'] > latest_version
+
+          latest_version = matching_url['version']
+          canonical_url = matching_url['url']
         end
+        page.data['canonical_url'] = canonical_url if canonical_url
 
         # If a page has a canonical URL and is not the /latest/ page,
         # we don't want it in the sitemap or indexable by Google
@@ -138,6 +156,17 @@ def set_canonical_and_noindex(site, all_pages) # rubocop:disable Metrics/AbcSize
       end
     end
 
+    def resolve_moved_url(url)
+      resolved_url = nil
+      loop do
+        url = @moved_pages[url]
+        break unless url
+
+        resolved_url = url
+      end
+      resolved_url
+    end
+
     def versioned_page?(url_segments)
       /^\d+\.\d+\.x$/.match(url_segments[1]) || url_segments[1] == 'latest'
     end

app/moved_urls.yml

@@ -0,0 +1,2 @@
+---
+/gateway-oss/VERSION/configuration/: "/gateway/VERSION/reference/configuration/"

docs/seo.md

@@ -0,0 +1,28 @@
+# SEO
+
+The canonical URL for each page is automatically set by the `canonical_url_generator` plugin. It works by looking for a URL that matches the current page, but where the version is higher than the current URL. As an example:
+
+1. `/gateway/2.3.x/configuration` would have a canonical URL of `/gateway/2.5.x/configuration`. `2.5.x` is the last version in which this URL existed.
+
+2. In `2.6.x` the page moved to `/gateway/2.6.x/reference/configuration`, which means the canonical URL would normally be `/gateway/2.8.x/reference/configuration`.
+
+3. _However_, we have a special `/latest/` URL which is always the latest version, so the canonical URL for the 2.6.x link above would actually be `/gateway/latest/reference/configuration`.
+
+## Tracking file renames
+
+It is possible to track pages through renames using the `moved_urls.yml` file. This is a key:value file that contains the old URL and the URL that it should be mapped to. e.g.
+
+```yaml
+---
+/gateway-oss/VERSION/configuration/: "/gateway/VERSION/reference/configuration/" # 2.5.x
+```
+
+It is possible for a URL to be forwarded multiple times. In this instance, the final URL will be set as canonical on all pages rather than creating a canonical chain:
+
+```yaml
+---
+/gateway-oss/VERSION/configuration/: "/foo/bar/" # 2.5.x
+/foo/bar/: "/gateway/VERSION/reference/configuration/" # 4.2.x
+```
+
+Each line ends with a comment. This is the latest version that uses the source URL. This is useful to keep track of, as it allows us to remove entries from `moved_urls.yml` when archiving old content. In this example, when `2.5.x` is archived we can remove the `/gateway-oss/VERSION/configuration/` canonical redirect.

tests/seo.test.js

@@ -40,6 +40,11 @@ test.describe("Canonical links", () => {
       src: "/hub/",
       href: "/hub/",
     },
+    {
+      title: "page using moved_urls.yml to track renamed files",
+      src: "/gateway-oss/2.5.x/configuration/",
+      href: "/gateway/latest/reference/configuration/",
+    },
   ].forEach((t) => {
     test(t.title, async ({ page }) => {
       await page.goto(t.src);