Mark classic queue mirroring as deprecated

[Why]
Classic queue mirroring will be removed in RabbitMQ 4.0. Quorum queues
provide a better safer alternative. Non-replicated classic queues remain
supported.

[How]
Classic queue mirroring is marked as deprecated in the code using the
Deprecated features subsystem (based on feature flags). See #7390 for a
description of that subsystem.

To test RabbitMQ behavior as if the feature was removed, the following
configuration setting can be used:
deprecated_features.permit.classic_queue_mirroring = false

To turn off classic queue mirroring, there must be no classic mirrored
queues declared and no HA policy defined.

Once classic queue mirroring is turned off, users will not be able to
declare HA policies. Trying to do that from the CLI or the management
API will be rejected with a warning in the logs. This impacts clustering
too: a node with classic queue mirroring turned off will only cluster
with another node which has no HA policy or has classic queue mirroring
turned off.

Note that given the marketing calendar, the deprecated feature will go
directly from "permitted by default" to "removed" in RabbitMQ 4.0. It
won't go through the gradual deprecation process.

V2: Renamed the deprecated feature from `classic_mirrored_queues` to
    `classic_queue_mirroring` to better reflect the intention. Otherwise
    it could be unclear is only the mirroring property is
    deprecated/removed or classic queues entirely.

Author: dumbbell

deps/rabbit/src/rabbit.erl

@@ -193,12 +193,6 @@
                    [{description, "core initialized"},
                     {requires,    kernel_ready}]}).
 
--rabbit_boot_step({deprecate_cmqs,
-                   [{description, "checks whether mirrored queues are disabled"},
-                    {mfa, {rabbit_mirror_queue_misc, prevent_startup_when_mirroring_is_disabled_but_configured, []}},
-                    {requires, [database]},
-                    {enables, [recovery]}]}).
-
 -rabbit_boot_step({recovery,
                    [{description, "exchange, queue and binding recovery"},
                     {mfa,         {rabbit, recover, []}},

deps/rabbit/src/rabbit_mirror_queue_misc.erl

@@ -9,6 +9,8 @@
 -behaviour(rabbit_policy_validator).
 -behaviour(rabbit_policy_merge_strategy).
 
+-include_lib("stdlib/include/assert.hrl").
+
 -include("amqqueue.hrl").
 
 -export([remove_from_queue/3, on_vhost_up/1, add_mirrors/3,
@@ -26,7 +28,8 @@
 
 -export([get_replicas/1, transfer_leadership/2, migrate_leadership_to_existing_replica/2]).
 
--export([prevent_startup_when_mirroring_is_disabled_but_configured/0]).
+%% Deprecated feature callback.
+-export([are_cmqs_used/1]).
 
 %% for testing only
 -export([module/1]).
@@ -35,6 +38,41 @@
 
 -define(HA_NODES_MODULE, rabbit_mirror_queue_mode_nodes).
 
+-rabbit_deprecated_feature(
+   {classic_queue_mirroring,
+    #{deprecation_phase => permitted_by_default,
+      messages =>
+      #{when_permitted =>
+        "Classic mirrored queues are deprecated.\n"
+        "By default, they can still be used for now.\n"
+        "Their use will not be permitted by default in the next minor"
+        "RabbitMQ version (if any) and they will be removed from "
+        "RabbitMQ 4.0.0.\n"
+        "To continue using classic mirrored queues when they are not "
+        "permitted by default, set the following parameter in your "
+        "configuration:\n"
+        "    \"deprecated_features.permit.classic_queue_mirroring = true\"\n"
+        "To test RabbitMQ as if they were removed, set this in your "
+        "configuration:\n"
+        "    \"deprecated_features.permit.classic_queue_mirroring = false\"",
+
+        when_denied =>
+        "Classic mirrored queues are deprecated.\n"
+        "Their use is not permitted per the configuration (overriding the "
+        "default, which is permitted):\n"
+        "    \"deprecated_features.permit.classic_queue_mirroring = false\"\n"
+        "Their use will not be permitted by default in the next minor "
+        "RabbitMQ version (if any) and they will be removed from "
+        "RabbitMQ 4.0.0.\n"
+        "To continue using classic mirrored queues when they are not "
+        "permitted by default, set the following parameter in your "
+        "configuration:\n"
+        "    \"deprecated_features.permit.classic_queue_mirroring = true\""
+       },
+      doc_url => "https://blog.rabbitmq.com/posts/2021/08/4.0-deprecation-announcements/#removal-of-classic-queue-mirroring",
+      callbacks => #{is_feature_used => {?MODULE, are_cmqs_used}}
+     }}).
+
 -rabbit_boot_step(
    {?MODULE,
     [{description, "HA policy validation"},
@@ -730,62 +768,32 @@ maybe_drop_master_after_sync(Q) when ?is_amqqueue(Q) ->
 
 %%----------------------------------------------------------------------------
 
-mirroring_policies() ->
-    Policies = rabbit_policy:list_as_maps(),
-    OpPolicies = rabbit_policy:list_op_as_maps(),
-    IsMirroringPolicy = fun
-        (#{definition := #{<<"ha-mode">> := _}}) ->
-            true;
-        (_) ->
+are_cmqs_permitted() ->
+    FeatureName = classic_queue_mirroring,
+    rabbit_deprecated_features:is_permitted(FeatureName).
+
+are_cmqs_used(_) ->
+    try
+        LocalPolicies = rabbit_policy:list(),
+        LocalOpPolicies = rabbit_policy:list_op(),
+        has_ha_policies(LocalPolicies ++ LocalOpPolicies)
+    catch
+        exit:{aborted, {no_exists, _}} ->
+            %% This node is being initialized for the first time. Therefore it
+            %% must have no policies.
+            ?assert(rabbit_mnesia:is_running()),
             false
-    end,
-    {lists:filter(IsMirroringPolicy, Policies), lists:filter(IsMirroringPolicy, OpPolicies)}.
-
-report_vhosts_using_mirroring(MirrorPolicies, PolicyType) ->
-    PerVhost = lists:foldr(
-                 fun (#{vhost := VHost, name := Name}, Acc) ->
-                         maps:update_with(
-                           VHost,
-                           fun (Message) -> <<Name/binary, ", ", Message/binary>> end,
-                           <<"Virtual host ", VHost/binary, " has ", PolicyType/binary,
-                             " policies that configure mirroring: ", Name/binary>>,
-                           Acc)
-                 end,
-                 #{}, MirrorPolicies),
-    lists:foreach(
-      fun (Msg) -> rabbit_log:error("~ts", [Msg]) end,
-      maps:values(PerVhost)).
-
-prevent_startup_when_mirroring_is_disabled_but_configured() ->
-    case are_cmqs_permitted() of
-        true ->
-            ok;
-        false ->
-            Error = {error, {failed_to_deny_deprecated_features, [classic_mirrored_queues]}},
-            case mirroring_policies() of
-                {[], []} ->
-                    ok;
-                {Pols, []} ->
-                    report_vhosts_using_mirroring(Pols, <<"user">>),
-                    exit(Error);
-                {[], OpPols} ->
-                    report_vhosts_using_mirroring(OpPols, <<"operator">>),
-                    exit(Error);
-                {Pols, OpPols} ->
-                    report_vhosts_using_mirroring(Pols, <<"user">>),
-                    report_vhosts_using_mirroring(OpPols, <<"operator">>),
-                    exit(Error)
-            end
     end.
 
-are_cmqs_permitted() ->
-    %% FeatureName = classic_mirrored_queues,
-    %% rabbit_deprecated_features:is_permitted(FeatureName).
-    case application:get_env(rabbit, permitted_deprecated_features) of
-        {ok, #{classic_mirrored_queues := false}} ->
-            false;
-        _ -> true
-    end.
+has_ha_policies(Policies) ->
+    lists:any(
+      fun(Policy) ->
+              KeyList = proplists:get_value(definition, Policy),
+              does_policy_configure_cmq(KeyList)
+      end, Policies).
+
+does_policy_configure_cmq(KeyList) ->
+    lists:keymember(<<"ha-mode">>, 1, KeyList).
 
 validate_policy(KeyList) ->
     Mode = proplists:get_value(<<"ha-mode">>, KeyList, none),
@@ -801,7 +809,12 @@ validate_policy(KeyList) ->
         {_, none, none, none, none, none, none} ->
             ok;
         {false, _, _, _, _, _, _} ->
-            {error, "Classic queue mirroring is disabled via node configuration", []};
+            %% If the policy configures classic mirrored queues and this
+            %% feature is disabled, we consider this policy not valid and deny
+            %% it.
+            FeatureName = classic_queue_mirroring,
+            Warning = rabbit_deprecated_features:get_warning(FeatureName),
+            {error, "~ts", [Warning]};
         {_, none, _, _, _, _, _} ->
             {error, "ha-mode must be specified to specify ha-params, "
              "ha-sync-mode or ha-promote-on-shutdown", []};

deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets

@@ -929,9 +929,9 @@ credential_validator.regexp = ^abc\\d+",
   %%
 
   {deprecated_features_cmq,
-   "deprecated_features.permit.classic_mirrored_queues = false",
+   "deprecated_features.permit.classic_queue_mirroring = false",
    [{rabbit, [
-      {permit_deprecated_features, #{classic_mirrored_queues => false}}
+      {permit_deprecated_features, #{classic_queue_mirroring => false}}
      ]}],
    []},
 

deps/rabbit/test/rabbitmq_4_0_deprecations_SUITE.erl

@@ -26,7 +26,10 @@
          when_global_qos_is_not_permitted_from_conf/1,
 
          join_when_ram_node_type_is_permitted_by_default/1,
-         join_when_ram_node_type_is_not_permitted_from_conf/1
+         join_when_ram_node_type_is_not_permitted_from_conf/1,
+
+         set_policy_when_cmq_is_permitted_by_default/1,
+         set_policy_when_cmq_is_not_permitted_from_conf/1
         ]).
 
 suite() ->
@@ -45,7 +48,10 @@ groups() ->
        when_global_qos_is_not_permitted_from_conf]},
      {ram_node_type, [],
       [join_when_ram_node_type_is_permitted_by_default,
-       join_when_ram_node_type_is_not_permitted_from_conf]}
+       join_when_ram_node_type_is_not_permitted_from_conf]},
+     {classic_queue_mirroring, [],
+      [set_policy_when_cmq_is_permitted_by_default,
+       set_policy_when_cmq_is_not_permitted_from_conf]}
     ].
 
 %% -------------------------------------------------------------------
@@ -67,6 +73,8 @@ init_per_group(global_qos, Config) ->
 init_per_group(ram_node_type, Config) ->
     rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 2},
                                           {rmq_nodes_clustered, false}]);
+init_per_group(classic_queue_mirroring, Config) ->
+    rabbit_ct_helpers:set_config(Config, {rmq_nodes_count, 1});
 init_per_group(_Group, Config) ->
     Config.
 
@@ -87,6 +95,14 @@ init_per_testcase(
                 {rabbit,
                  [{permit_deprecated_features, #{ram_node_type => false}}]}),
     init_per_testcase1(Testcase, Config1);
+init_per_testcase(
+  set_policy_when_cmq_is_not_permitted_from_conf = Testcase, Config) ->
+    Config1 = rabbit_ct_helpers:merge_app_env(
+                Config,
+                {rabbit,
+                 [{permit_deprecated_features,
+                   #{classic_queue_mirroring => false}}]}),
+    init_per_testcase1(Testcase, Config1);
 init_per_testcase(Testcase, Config) ->
     init_per_testcase1(Testcase, Config).
 
@@ -233,3 +249,46 @@ get_disc_nodes(Config, Node) ->
     lists:sort(
       rabbit_ct_broker_helpers:rpc(
         Config, Node, rabbit_mnesia, cluster_nodes, [disc])).
+
+%% -------------------------------------------------------------------
+%% Classic queue mirroring.
+%% -------------------------------------------------------------------
+
+set_policy_when_cmq_is_permitted_by_default(Config) ->
+    ?assertEqual(
+       ok,
+       rabbit_ct_broker_helpers:set_ha_policy(
+         Config, 0, <<".*">>, <<"all">>)),
+
+    [NodeA] = rabbit_ct_broker_helpers:get_node_configs(
+                Config, nodename),
+
+    %% Change the advanced configuration file to turn off classic queue
+    %% mirroring.
+    ConfigFilename0 = rabbit_ct_broker_helpers:get_node_config(
+                        Config, NodeA, erlang_node_config_filename),
+    ConfigFilename = ConfigFilename0 ++ ".config",
+    {ok, [ConfigContent0]} = file:consult(ConfigFilename),
+    ConfigContent1 = rabbit_ct_helpers:merge_app_env_in_erlconf(
+                       ConfigContent0,
+                       {rabbit, [{permit_deprecated_features,
+                                  #{classic_queue_mirroring => false}}]}),
+    ConfigContent2 = lists:flatten(io_lib:format("~p.~n", [ConfigContent1])),
+    ok = file:write_file(ConfigFilename, ConfigContent2),
+    ?assertEqual({ok, [ConfigContent1]}, file:consult(ConfigFilename)),
+
+    %% Restart the node and see if it was correctly converted to a disc node.
+    ok = rabbit_control_helper:command(stop_app, NodeA),
+    ?assertMatch(
+       {error, 69,
+        <<"Error:\n{:rabbit, {{:failed_to_deny_deprecated_features, "
+        "[:classic_queue_mirroring]}", _/binary>>},
+       rabbit_control_helper:command(start_app, NodeA)).
+
+set_policy_when_cmq_is_not_permitted_from_conf(Config) ->
+    ?assertError(
+       {badmatch,
+        {error_string,
+         "Validation failed\n\nClassic mirrored queues are deprecated." ++ _}},
+       rabbit_ct_broker_helpers:set_ha_policy(
+         Config, 0, <<".*">>, <<"all">>)).