From cca64e8ed608a68a96861b97589d4fd24832217b Mon Sep 17 00:00:00 2001 From: Karl Nilsson Date: Mon, 11 Dec 2023 13:12:04 +0000 Subject: [PATCH] Add new local random exchange type. This exchange type will only bind classic queues and will only return routes for queues that are local to the publishing connection. If more than one queue is bound it will make a random choice of the locally bound queues. This exchange type is suitable as a component in systems that run highly available low-latency RPC workloads. Co-authored-by: Marcial Rosales --- deps/rabbit/BUILD.bazel | 8 + deps/rabbit/app.bzl | 13 +- .../src/rabbit_exchange_type_local_random.erl | 116 ++++++++++ deps/rabbit/test/quorum_queue_SUITE.erl | 5 +- .../rabbit_local_random_exchange_SUITE.erl | 205 ++++++++++++++++++ moduleindex.yaml | 1 + 6 files changed, 345 insertions(+), 3 deletions(-) create mode 100644 deps/rabbit/src/rabbit_exchange_type_local_random.erl create mode 100644 deps/rabbit/test/rabbit_local_random_exchange_SUITE.erl diff --git a/deps/rabbit/BUILD.bazel b/deps/rabbit/BUILD.bazel index 66e08da658ad..1915327b978d 100644 --- a/deps/rabbit/BUILD.bazel +++ b/deps/rabbit/BUILD.bazel @@ -1062,6 +1062,14 @@ rabbitmq_integration_suite( ], ) +rabbitmq_integration_suite( + name = "rabbit_local_random_exchange_SUITE", + size = "small", + additional_beam = [ + ":test_queue_utils_beam", + ], +) + rabbitmq_integration_suite( name = "rabbit_direct_reply_to_prop_SUITE", size = "medium", diff --git a/deps/rabbit/app.bzl b/deps/rabbit/app.bzl index 34103a3bfafe..0c9459bf358b 100644 --- a/deps/rabbit/app.bzl +++ b/deps/rabbit/app.bzl @@ -131,6 +131,7 @@ def all_beam_files(name = "all_beam_files"): "src/rabbit_exchange_type_fanout.erl", "src/rabbit_exchange_type_headers.erl", "src/rabbit_exchange_type_invalid.erl", + "src/rabbit_exchange_type_local_random.erl", "src/rabbit_exchange_type_topic.erl", "src/rabbit_feature_flags.erl", "src/rabbit_ff_controller.erl", @@ -391,6 +392,7 @@ def all_test_beam_files(name = "all_test_beam_files"): "src/rabbit_exchange_type_fanout.erl", "src/rabbit_exchange_type_headers.erl", "src/rabbit_exchange_type_invalid.erl", + "src/rabbit_exchange_type_local_random.erl", "src/rabbit_exchange_type_topic.erl", "src/rabbit_feature_flags.erl", "src/rabbit_ff_controller.erl", @@ -670,6 +672,7 @@ def all_srcs(name = "all_srcs"): "src/rabbit_exchange_type_fanout.erl", "src/rabbit_exchange_type_headers.erl", "src/rabbit_exchange_type_invalid.erl", + "src/rabbit_exchange_type_local_random.erl", "src/rabbit_exchange_type_topic.erl", "src/rabbit_feature_flags.erl", "src/rabbit_ff_controller.erl", @@ -2048,7 +2051,6 @@ def test_suite_beam_files(name = "test_suite_beam_files"): erlc_opts = "//:test_erlc_opts", deps = ["//deps/amqp_client:erlang_app"], ) - erlang_bytecode( name = "test_event_recorder_beam", testonly = True, @@ -2129,3 +2131,12 @@ def test_suite_beam_files(name = "test_suite_beam_files"): erlc_opts = "//:test_erlc_opts", deps = ["//deps/amqp_client:erlang_app"], ) + erlang_bytecode( + name = "rabbit_local_random_exchange_SUITE_beam_files", + testonly = True, + srcs = ["test/rabbit_local_random_exchange_SUITE.erl"], + outs = ["test/rabbit_local_random_exchange_SUITE.beam"], + app_name = "rabbit", + erlc_opts = "//:test_erlc_opts", + deps = ["//deps/amqp_client:erlang_app"], + ) diff --git a/deps/rabbit/src/rabbit_exchange_type_local_random.erl b/deps/rabbit/src/rabbit_exchange_type_local_random.erl new file mode 100644 index 000000000000..db9b37475fdc --- /dev/null +++ b/deps/rabbit/src/rabbit_exchange_type_local_random.erl @@ -0,0 +1,116 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2023 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(rabbit_exchange_type_local_random). +-behaviour(rabbit_exchange_type). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-rabbit_feature_flag({?MODULE, + #{desc => "Local random exchange", + stability => stable + }}). + +-rabbit_boot_step({?MODULE, + [{description, "exchange type local random"}, + {mfa, {rabbit_registry, register, + [exchange, <<"x-local-random">>, ?MODULE]}}, + {requires, rabbit_registry}, + {enables, kernel_ready} + ]}). + +-export([add_binding/3, + assert_args_equivalence/2, + create/2, + delete/2, + policy_changed/2, + description/0, + recover/2, + remove_bindings/3, + validate_binding/2, + route/3, + serialise_events/0, + validate/1, + info/1, + info/2 + ]). + +description() -> + [{name, <<"x-local-random">>}, + {description, <<"Picks one random local binding (queue) to route via (to).">>}]. + +route(#exchange{name = Name}, _Msg, _Opts) -> + Matches = rabbit_router:match_routing_key(Name, [<<>>]), + case lists:filter(fun filter_local_queue/1, Matches) of + [] -> + []; + [_] = One -> + One; + LocalMatches -> + Rand = rand:uniform(length(LocalMatches)), + [lists:nth(Rand, LocalMatches)] + end. + +info(_X) -> []. +info(_X, _) -> []. +serialise_events() -> false. +validate(_X) -> + case rabbit_feature_flags:is_enabled(?MODULE) of + true -> + ok; + false -> + rabbit_misc:amqp_error( + precondition_failed, + "x-local-random exchange feature not available", [], + 'exchange.declare') + end. + +create(_Serial, _X) -> ok. +recover(_X, _Bs) -> ok. +delete(_Serial, _X) -> ok. +policy_changed(_X1, _X2) -> ok. +add_binding(_Serial, _X, _B) -> ok. +remove_bindings(_Serial, _X, _Bs) -> ok. + +validate_binding(_X, #binding{destination = Dest, + key = <<>>}) -> + case rabbit_amqqueue:lookup(Dest) of + {ok, Q} -> + case amqqueue:get_type(Q) of + rabbit_classic_queue -> + ok; + Type -> + {error, {binding_invalid, + "Queue type ~ts not valid for this exchange type", + [Type]}} + end; + _ -> + {error, {binding_invalid, + "Destination not found", + []}} + end; +validate_binding(_X, #binding{key = BKey}) -> + {error, {binding_invalid, + "Non empty binding '~s' key not permitted", + [BKey]}}. + +assert_args_equivalence(X, Args) -> + rabbit_exchange:assert_args_equivalence(X, Args). + +filter_local_queue(QName) -> + %% TODO: introduce lookup function that _only_ gets the pid + case rabbit_amqqueue:lookup(QName) of + {ok, Q} -> + case amqqueue:get_pid(Q) of + Pid when is_pid(Pid) andalso + node(Pid) =:= node() -> + is_process_alive(Pid); + _ -> + false + end; + _ -> + false + end. diff --git a/deps/rabbit/test/quorum_queue_SUITE.erl b/deps/rabbit/test/quorum_queue_SUITE.erl index 789ca6d5a6b6..726f9bb378dd 100644 --- a/deps/rabbit/test/quorum_queue_SUITE.erl +++ b/deps/rabbit/test/quorum_queue_SUITE.erl @@ -225,7 +225,8 @@ init_per_group(Group, Config) -> Config1 = rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, ClusterSize}, {rmq_nodename_suffix, Group}, - {tcp_ports_base}]), + {tcp_ports_base, {skip_n_nodes, ClusterSize}} + ]), Config1b = rabbit_ct_helpers:set_config(Config1, [{net_ticktime, 10}]), Ret = rabbit_ct_helpers:run_steps(Config1b, [fun merge_app_env/1 ] ++ @@ -263,7 +264,7 @@ init_per_testcase(Testcase, Config) when Testcase == reconnect_consumer_and_publ Config2 = rabbit_ct_helpers:set_config(Config1, [{rmq_nodes_count, 3}, {rmq_nodename_suffix, Testcase}, - {tcp_ports_base}, + {tcp_ports_base, {skip_n_nodes, 3}}, {queue_name, Q}, {alt_queue_name, <>} ]), diff --git a/deps/rabbit/test/rabbit_local_random_exchange_SUITE.erl b/deps/rabbit/test/rabbit_local_random_exchange_SUITE.erl new file mode 100644 index 000000000000..a2e57f476171 --- /dev/null +++ b/deps/rabbit/test/rabbit_local_random_exchange_SUITE.erl @@ -0,0 +1,205 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2023 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(rabbit_local_random_exchange_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + routed_to_one_local_queue_test, + no_route + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test suite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:set_config(Config, + [ + {rmq_nodename_suffix, ?MODULE}, + {rmq_nodes_count, 3}, + {tcp_ports_base, {skip_n_nodes, 3}} + ]), + rabbit_ct_helpers:run_setup_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps( + Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag( + Config, rabbit_exchange_type_local_random) of + ok -> + TestCaseName = rabbit_ct_helpers:config_to_testcase_name(Config, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, {test_resource_name, + re:replace(TestCaseName, "/", "-", + [global, {return, list}])}), + rabbit_ct_helpers:testcase_started(Config1, Testcase); + Res -> + {skip, Res} + end. + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +routed_to_one_local_queue_test(Config) -> + E = make_exchange_name(Config, "0"), + declare_exchange(Config, E), + %% declare queue on the first two nodes: 0, 1 + QueueNames = declare_and_bind_queues(Config, 2, E), + %% publish message on node 1 + publish(Config, E, 0), + publish(Config, E, 1), + %% message should arrive to queue on node 1 + run_on_node(Config, 0, + fun(Chan) -> + assert_queue_size(Config, Chan, 1, lists:nth(1, QueueNames)) + end), + run_on_node(Config, 0, + fun(Chan) -> + assert_queue_size(Config, Chan, 1, lists:nth(2, QueueNames)) + end), + delete_exchange_and_queues(Config, E, QueueNames), + ok. + +no_route(Config) -> + + E = make_exchange_name(Config, "0"), + declare_exchange(Config, E), + %% declare queue on nodes 0, 1 + QueueNames = declare_and_bind_queues(Config, 2, E), + %% publish message on node 2 + publish_expect_return(Config, E, 2), + %% message should arrive to any of the other nodes. Total size among all queues is 1 + delete_exchange_and_queues(Config, E, QueueNames), + ok. + +delete_exchange_and_queues(Config, E, QueueNames) -> + run_on_node(Config, 0, + fun(Chan) -> + amqp_channel:call(Chan, #'exchange.delete'{exchange = E }), + [amqp_channel:call(Chan, #'queue.delete'{queue = Q }) + || Q <- QueueNames] + end). +publish(Config, E, Node) -> + run_on_node(Config, Node, + fun(Chan) -> + amqp_channel:call(Chan, #'confirm.select'{}), + amqp_channel:call(Chan, + #'basic.publish'{exchange = E, routing_key = rnd()}, + #amqp_msg{props = #'P_basic'{}, payload = <<>>}), + amqp_channel:wait_for_confirms_or_die(Chan) + end). + +publish_expect_return(Config, E, Node) -> + run_on_node(Config, Node, + fun(Chan) -> + amqp_channel:register_return_handler(Chan, self()), + amqp_channel:call(Chan, + #'basic.publish'{exchange = E, + mandatory = true, + routing_key = rnd()}, + #amqp_msg{props = #'P_basic'{}, + payload = <<>>}), + receive + {#'basic.return'{}, _} -> + ok + after 5000 -> + flush(100), + ct:fail("no return received") + end + end). + +run_on_node(Config, Node, RunMethod) -> + {Conn, Chan} = rabbit_ct_client_helpers:open_connection_and_channel(Config, Node), + Return = RunMethod(Chan), + rabbit_ct_client_helpers:close_connection_and_channel(Conn, Chan), + Return. + +declare_exchange(Config, ExchangeName) -> + run_on_node(Config, 0, + fun(Chan) -> + #'exchange.declare_ok'{} = + amqp_channel:call(Chan, + #'exchange.declare'{exchange = ExchangeName, + type = <<"x-local-random">>, + auto_delete = false}) + end). + +declare_and_bind_queues(Config, NodeCount, E) -> + QueueNames = [make_queue_name(Config, Node) || Node <- lists:seq(0, NodeCount -1)], + [run_on_node(Config, Node, + fun(Chan) -> + declare_and_bind_queue(Chan, E, make_queue_name(Config, Node)) + end) || Node <- lists:seq(0, NodeCount -1)], + QueueNames. + +declare_and_bind_queue(Ch, E, Q) -> + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = Q}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = Q, + exchange = E, + routing_key = <<"">>}), + ok. + +assert_total_queue_size(_Config, Chan, ExpectedSize, ExpectedQueues) -> + Counts = [begin + #'queue.declare_ok'{message_count = M} = + amqp_channel:call(Chan, #'queue.declare'{queue = Q}), + M + end || Q <- ExpectedQueues], + ?assertEqual(ExpectedSize, lists:sum(Counts)). + +assert_queue_size(_Config, Chan, ExpectedSize, ExpectedQueue) -> + ct:log("assert_queue_size ~p ~p", [ExpectedSize, ExpectedQueue]), + #'queue.declare_ok'{message_count = M} = + amqp_channel:call(Chan, #'queue.declare'{queue = ExpectedQueue}), + ?assertEqual(ExpectedSize, M). + +rnd() -> + list_to_binary(integer_to_list(rand:uniform(1000000))). + +make_exchange_name(Config, Suffix) -> + B = rabbit_ct_helpers:get_config(Config, test_resource_name), + erlang:list_to_binary("x-" ++ B ++ "-" ++ Suffix). +make_queue_name(Config, Node) -> + B = rabbit_ct_helpers:get_config(Config, test_resource_name), + erlang:list_to_binary("q-" ++ B ++ "-" ++ integer_to_list(Node)). + +flush(T) -> + receive X -> + ct:pal("flushed ~p", [X]), + flush(T) + after T -> + ok + end. diff --git a/moduleindex.yaml b/moduleindex.yaml index 4990c9ae82ae..9673bcc804c8 100755 --- a/moduleindex.yaml +++ b/moduleindex.yaml @@ -647,6 +647,7 @@ rabbit: - rabbit_exchange_type_fanout - rabbit_exchange_type_headers - rabbit_exchange_type_invalid +- rabbit_exchange_type_local_random - rabbit_exchange_type_topic - rabbit_feature_flags - rabbit_ff_controller