Use recon_alloc to get memory info on Win32, /proc on Linux, ps for other OS types

LICENSE-BSD-recon

@@ -0,0 +1,27 @@
+Copyright (c) 2012-2017, Frédéric Trottier-Hébert
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+  Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+  The names of its contributors may not be used to endorse or promote
+  products derived from this software without specific prior written
+  permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile

@@ -19,6 +19,7 @@ define PROJECT_APP_EXTRA_KEYS
 	  ]}
 endef
 
+DEPS = recon
 LOCAL_DEPS = compiler syntax_tools xmerl
 
 # FIXME: Use erlang.mk patched for RabbitMQ, while waiting for PRs to be

src/app_utils.erl

@@ -50,7 +50,7 @@ stop_applications(Apps) ->
 
 start_applications(Apps, ErrorHandler) ->
     manage_applications(fun lists:foldl/3,
-                        fun application:start/1,
+                        fun application:ensure_all_started/1,
                         fun application:stop/1,
                         already_started,
                         ErrorHandler,
@@ -62,7 +62,7 @@ stop_applications(Apps, ErrorHandler) ->
                             rabbit_log:info("Stopping application '~s'", [App]),
                             application:stop(App)
                         end,
-                        fun application:start/1,
+                        fun application:ensure_all_started/1,
                         not_started,
                         ErrorHandler,
                         Apps).
@@ -116,6 +116,9 @@ manage_applications(Iterate, Do, Undo, SkipError, ErrorHandler, Apps) ->
     Iterate(fun (App, Acc) ->
                     case Do(App) of
                         ok -> [App | Acc];
+                        {ok, []} -> Acc;
+                        {ok, [App]} -> [App | Acc];
+                        {ok, StartedApps} -> StartedApps ++ Acc;
                         {error, {SkipError, _}} -> Acc;
                         {error, Reason} ->
                             lists:foreach(Undo, Acc),

src/vm_memory_monitor.erl

@@ -50,8 +50,11 @@
                 timeout,
                 timer,
                 alarmed,
-                alarm_funs
-               }).
+                alarm_funs,
+                os_type = undefined,
+                os_pid  = undefined,
+                page_size = undefined,
+                proc_file = undefined}).
 
 -include("rabbit_memory.hrl").
 
@@ -130,86 +133,38 @@ get_memory_use(ratio) ->
 %% be equal to the total size of all pages mapped to the emulator,
 %% according to http://erlang.org/doc/man/erlang.html#memory-0
 %% erlang:memory(total) under-reports memory usage by around 20%
+%%
+%% Win32 Note: 3.6.12 shipped with code that used wmic.exe to get the
+%% WorkingSetSize value for the running erl.exe process. Unfortunately
+%% even with a moderate invocation rate of 1 ops/second that uses more
+%% CPU resources than some Windows users are willing to tolerate.
+%% See rabbitmq/rabbitmq-server#1343 and rabbitmq/rabbitmq-common#224
+%% for details.
 -spec get_process_memory() -> Bytes :: integer().
 get_process_memory() ->
-    case get_memory_calculation_strategy() of
-        rss ->
-            case get_system_process_resident_memory() of
-                {ok, MemInBytes} ->
-                    MemInBytes;
-                {error, Reason}  ->
-                    rabbit_log:debug("Unable to get system memory used. Reason: ~p."
-                                     " Falling back to erlang memory reporting",
-                                     [Reason]),
-                    erlang:memory(total)
-            end;
-        erlang ->
-            erlang:memory(total)
+    try
+        {ProcMem, _} = get_memory_use(bytes),
+        ProcMem
+    catch exit:{noproc, Error} ->
+        rabbit_log:warning("Memory monitor process not yet started: ~p~n", [Error]),
+        get_process_memory_uncached()
     end.
 
 -spec get_memory_calculation_strategy() -> rss | erlang.
 get_memory_calculation_strategy() ->
-    case rabbit_misc:get_env(rabbit, vm_memory_calculation_strategy, rss) of
-        erlang ->
-            erlang;
-        rss ->
-            rss;
+    case rabbit_misc:get_env(rabbit, vm_memory_calculation_strategy, allocated) of
+        allocated -> allocated;
+        erlang -> erlang;
+        legacy -> erlang; %% backwards compatibility
+        rss -> rss;
         UnsupportedValue ->
             rabbit_log:warning(
               "Unsupported value '~p' for vm_memory_calculation_strategy. "
-              "Supported values: (rss|erlang). "
-              "Defaulting to 'rss'",
+              "Supported values: (allocated|erlang|legacy|rss). "
+              "Defaulting to 'allocated'",
               [UnsupportedValue]
             ),
-            rss
-    end.
-
--spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
-get_system_process_resident_memory() ->
-    try
-        get_system_process_resident_memory(os:type())
-    catch _:Error ->
-            {error, {"Failed to get process resident memory", Error}}
-    end.
-
-get_system_process_resident_memory({unix,darwin}) ->
-    get_ps_memory();
-
-get_system_process_resident_memory({unix, linux}) ->
-    get_ps_memory();
-
-get_system_process_resident_memory({unix,freebsd}) ->
-    get_ps_memory();
-
-get_system_process_resident_memory({unix,openbsd}) ->
-    get_ps_memory();
-
-get_system_process_resident_memory({win32,_OSname}) ->
-    %% Note: 3.6.12 shipped with code that used wmic.exe to get the
-    %% WorkingSetSize value for the running erl.exe process. Unfortunately
-    %% even with a moderate invocation rate of 1 ops/second that uses more
-    %% CPU resources than some Windows users are willing to tolerate.
-    %% See rabbitmq/rabbitmq-server#1343 for details.
-    {ok, erlang:memory(total)};
-
-get_system_process_resident_memory({unix, sunos}) ->
-    get_ps_memory();
-
-get_system_process_resident_memory({unix, aix}) ->
-    get_ps_memory();
-
-get_system_process_resident_memory(_OsType) ->
-    {error, not_implemented_for_os}.
-
-get_ps_memory() ->
-    OsPid = os:getpid(),
-    Cmd = "ps -p " ++ OsPid ++ " -o rss=",
-    CmdOutput = os:cmd(Cmd),
-    case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
-        {match, [Match]} ->
-            {ok, list_to_integer(Match) * 1024};
-        _ ->
-            {error, {unexpected_output_from_command, Cmd, CmdOutput}}
+            allocated
     end.
 
 %%----------------------------------------------------------------------------
@@ -226,11 +181,12 @@ start_link(MemFraction, AlarmSet, AlarmClear) ->
 
 init([MemFraction, AlarmFuns]) ->
     TRef = erlang:send_after(?DEFAULT_MEMORY_CHECK_INTERVAL, self(), update),
-    State = #state { timeout    = ?DEFAULT_MEMORY_CHECK_INTERVAL,
-                     timer      = TRef,
-                     alarmed    = false,
-                     alarm_funs = AlarmFuns },
-    {ok, set_mem_limits(State, MemFraction)}.
+    State0 = #state{timeout    = ?DEFAULT_MEMORY_CHECK_INTERVAL,
+                    timer      = TRef,
+                    alarmed    = false,
+                    alarm_funs = AlarmFuns},
+    State1 = init_state_by_os(State0),
+    {ok, set_mem_limits(State1, MemFraction)}.
 
 handle_call(get_vm_memory_high_watermark, _From,
             #state{memory_config_limit = MemLimit} = State) ->
@@ -282,6 +238,53 @@ code_change(_OldVsn, State, _Extra) ->
 %%----------------------------------------------------------------------------
 %% Server Internals
 %%----------------------------------------------------------------------------
+
+get_process_memory_uncached() ->
+    TmpState = init_state_by_os(#state{}),
+    TmpState#state.process_memory.
+
+update_process_memory(State) ->
+    Strategy = get_memory_calculation_strategy(),
+    {ok, ProcMem} = get_process_memory_using_strategy(Strategy, State),
+    State#state{process_memory = ProcMem}.
+
+init_state_by_os(State = #state{os_type = undefined}) ->
+    OsType = os:type(),
+    OsPid = os:getpid(),
+    init_state_by_os(State#state{os_type = OsType, os_pid = OsPid});
+init_state_by_os(State0 = #state{os_type = {unix, linux}, os_pid = OsPid}) ->
+    PageSize = get_linux_pagesize(),
+    ProcFile = io_lib:format("/proc/~s/statm", [OsPid]),
+    State1 = State0#state{page_size = PageSize, proc_file = ProcFile},
+    update_process_memory(State1);
+init_state_by_os(State) ->
+    update_process_memory(State).
+
+get_process_memory_using_strategy(rss, #state{os_type = {unix, linux},
+                                              page_size = PageSize,
+                                              proc_file = ProcFile}) ->
+    Data = read_proc_file(ProcFile),
+    [_|[RssPagesStr|_]] = string:split(Data, " ", all),
+    ProcMem = list_to_integer(RssPagesStr) * PageSize,
+    {ok, ProcMem};
+get_process_memory_using_strategy(rss, #state{os_type = {unix, _},
+                                              os_pid = OsPid}) ->
+    Cmd = "ps -p " ++ OsPid ++ " -o rss=",
+    CmdOutput = os:cmd(Cmd),
+    case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
+        {match, [Match]} ->
+            ProcMem = list_to_integer(Match) * 1024,
+            {ok, ProcMem};
+        _ ->
+            {error, {unexpected_output_from_command, Cmd, CmdOutput}}
+    end;
+get_process_memory_using_strategy(rss, _State) ->
+    {ok, recon_alloc:memory(allocated)};
+get_process_memory_using_strategy(allocated, _State) ->
+    {ok, recon_alloc:memory(allocated)};
+get_process_memory_using_strategy(erlang, _State) ->
+    {ok, erlang:memory(total)}.
+
 get_total_memory_from_os() ->
     try
         get_total_memory(os:type())
@@ -374,19 +377,20 @@ parse_mem_limit(MemLimit) ->
     ),
     ?DEFAULT_VM_MEMORY_HIGH_WATERMARK.
 
-internal_update(State = #state { memory_limit = MemLimit,
-                                 alarmed      = Alarmed,
-                                 alarm_funs   = {AlarmSet, AlarmClear} }) ->
-    ProcMem = get_process_memory(),
-    NewAlarmed = ProcMem > MemLimit,
+internal_update(State0 = #state{memory_limit = MemLimit,
+                                alarmed      = Alarmed,
+                                alarm_funs   = {AlarmSet, AlarmClear}}) ->
+    State1 = update_process_memory(State0),
+    ProcMem = State1#state.process_memory,
+    NewAlarmed = ProcMem  > MemLimit,
     case {Alarmed, NewAlarmed} of
         {false, true} -> emit_update_info(set, ProcMem, MemLimit),
                          AlarmSet({{resource_limit, memory, node()}, []});
         {true, false} -> emit_update_info(clear, ProcMem, MemLimit),
                          AlarmClear({resource_limit, memory, node()});
         _             -> ok
     end,
-    State #state {alarmed = NewAlarmed, process_memory = ProcMem}.
+    State1#state{alarmed = NewAlarmed}.
 
 emit_update_info(AlarmState, MemUsed, MemLimit) ->
     rabbit_log:info(
@@ -420,22 +424,33 @@ cmd(Command) ->
         _     -> os:cmd(Command)
     end.
 
+get_linux_pagesize() ->
+    CmdOutput = cmd("getconf PAGESIZE"),
+    case re:run(CmdOutput, "^[0-9]+", [{capture, first, list}]) of
+        {match, [Match]} -> list_to_integer(Match);
+        _ ->
+            rabbit_log:warning(
+                "Failed to get memory page size, using 4096:~n~p~n",
+                [CmdOutput]),
+            4096
+    end.
+
 %% get_total_memory(OS) -> Total
 %% Windows and Freebsd code based on: memsup:get_memory_usage/1
 %% Original code was part of OTP and released under "Erlang Public License".
 
-get_total_memory({unix,darwin}) ->
+get_total_memory({unix, darwin}) ->
     sysctl("hw.memsize");
 
-get_total_memory({unix,freebsd}) ->
+get_total_memory({unix, freebsd}) ->
     PageSize  = sysctl("vm.stats.vm.v_page_size"),
     PageCount = sysctl("vm.stats.vm.v_page_count"),
     PageCount * PageSize;
 
-get_total_memory({unix,openbsd}) ->
+get_total_memory({unix, openbsd}) ->
     sysctl("hw.usermem");
 
-get_total_memory({win32,_OSname}) ->
+get_total_memory({win32, _OSname}) ->
     [Result|_] = os_mon_sysinfo:get_mem_info(),
     {ok, [_MemLoad, TotPhys, _AvailPhys, _TotPage, _AvailPage, _TotV, _AvailV],
      _RestStr} =