Skip to content

Commit c85e780

Browse files
committed
process_collector: fill in virtual and resident memory values on macOS (prometheus#1600)
Unfortunately, these values aren't available from getrusage(2), or any other builtin Go API. Using cgo is one alternative. It's possible to conditionalize everything such that cgo can remain disabled on non-Darwin platforms, or even when cross-compiling Darwin executables on a non-Darwin platform (and stub in code that causes the metrics to not be exported). `CGO_ENABLED=1` is set by default on macOS, but unfortunately is off for the non-host architecture, even when gcc supports cross-compiling. (e.g. building with GOARCH=amd on an M2 mac skipped the cgo code.) I think that's too subtle of a distinction to rely on cgo. There's no builtin equivalent of `syscall.NewLazyDLL()` and `.NewProc()` on macOS that Go provides for Windows, so we're stuck with a 3rd party dependency. But it seems stable, maintained, ang getting a fair amount of usage. I'm avoiding their struct deserialization because these native structs are packed differently than the equivalent Go structs, which was causing bad values to be returned. The code is heavy with inline comments, and I tried keeping the type names the same as the C code to make it easier to search for them. I'm not sure that we need to do the `mach_vm_region()` call to adjust the `task_info()` values, because I've never seen that conditional evaluate to True on either amd64, arm64, or when amd64 is run under Rosetta. But this is what `ps(1)` does, and I think it's reasonable to try to match that unless somebody knows it's dead code. Signed-off-by: Matt Harbison <[email protected]>
1 parent 39e7c23 commit c85e780

6 files changed

+338
-1
lines changed

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
)
1818

1919
require (
20+
github.com/ebitengine/purego v0.7.1 // indirect
2021
github.com/jpillora/backoff v1.0.0 // indirect
2122
github.com/kr/pretty v0.3.1 // indirect
2223
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
66
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
77
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
88
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9+
github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA=
10+
github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
911
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
1012
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
1113
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright 2024 The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
//go:build amd64
15+
16+
package prometheus
17+
18+
// These are macros in xnu/osfmk/mach/shared_memory_server.h
19+
const (
20+
globalSharedTextSegment uint64 = 0x90000000
21+
22+
sharedTextRegionSize uint64 = 0x10000000
23+
sharedDataRegionSize uint64 = 0x10000000
24+
)
25+
26+
const (
27+
machTaskBasicInfoSizeOf = 48 // sizeof(struct mach_task_basic_info)
28+
vmRegionBasicInfo64SizeOf = 36 // sizeof(struct vm_region_basic_info_64)
29+
)
30+
31+
// Fundamental mach types defined in xnu/osfmk/mach/i386/vm_types.h. The integer_t and
32+
// __darwin_natural_t types are explicitly 32-bit here to help decoding into a structure,
33+
// where 'int' types are not supported. The __darwin_natural_t type is defined in
34+
// xnu/bsd/i386/_types.h.
35+
type (
36+
__darwin_natural_t = uint32 // typedef unsigned int __darwin_natural_t;
37+
integer_t = int32 // typedef int integer_t;
38+
natural_t = __darwin_natural_t // typedef __darwin_natural_t natural_t;
39+
40+
mach_vm_offset_t = uint64 // typedef uint64_t mach_vm_offset_t __kernel_ptr_semantics;
41+
mach_vm_size_t = uint64 // typedef uint64_t mach_vm_size_t;
42+
)
43+
44+
// Defined in xnu/osfmk/mach/i386/boolean.h, and explicitly 32-bit here to help decoding
45+
// into a structure, where 'int' types are not supported.
46+
type (
47+
boolean_t = uint32 // typedef unsigned int boolean_t;
48+
)
49+
50+
// Defined in xnu/osfmk/mach/i386/kern_return.h; see xnu/osfmk/mach/kern_return.h for
51+
// possible values.
52+
type (
53+
kern_return_t = int // typedef int kern_return_t;
54+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright 2024 The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
//go:build arm64
15+
16+
package prometheus
17+
18+
// These are macros in xnu/osfmk/mach/shared_memory_server.h. Note that __arm__
19+
// is not defined for 64-bit arm.
20+
const (
21+
globalSharedTextSegment uint64 = 0x90000000
22+
23+
sharedTextRegionSize uint64 = 0x10000000
24+
sharedDataRegionSize uint64 = 0x10000000
25+
)
26+
27+
const (
28+
machTaskBasicInfoSizeOf = 48 // sizeof(struct mach_task_basic_info)
29+
vmRegionBasicInfo64SizeOf = 36 // sizeof(struct vm_region_basic_info_64)
30+
)
31+
32+
// Fundamental mach types defined in xnu/osfmk/mach/arm/vm_types.h. The integer_t and
33+
// __darwin_natural_t types are explicitly 32-bit here to help decoding into a structure,
34+
// where 'int' types are not supported. The __darwin_natural_t type is defined in
35+
// xnu/bsd/arm/_types.h.
36+
type (
37+
__darwin_natural_t = uint32 // typedef unsigned int __darwin_natural_t;
38+
integer_t = int32 // typedef int integer_t;
39+
natural_t = __darwin_natural_t // typedef __darwin_natural_t natural_t;
40+
41+
mach_vm_offset_t = uint64 // typedef uint64_t mach_vm_offset_t __kernel_ptr_semantics;
42+
mach_vm_size_t = uint64 // typedef uint64_t mach_vm_size_t;
43+
)
44+
45+
// Defined in xnu/osfmk/mach/arm/boolean.h, and explicitly 32-bit here to help decoding
46+
// into a structure, where 'int' types are not supported.
47+
type (
48+
boolean_t = int32 // typedef int boolean_t;
49+
)
50+
51+
// Defined in xnu/osfmk/mach/arm/kern_return.h; see xnu/osfmk/mach/kern_return.h for
52+
// possible values.
53+
type (
54+
kern_return_t = int // typedef int kern_return_t;
55+
)

prometheus/process_collector_darwin.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,13 @@ func (c *processCollector) processCollect(ch chan<- Metric) {
8585
c.reportError(ch, c.cpuTotal, err)
8686
}
8787

88-
// TODO: publish c.vsize and c.rss values
88+
if info, err := getMemoryUsage(); err == nil {
89+
ch <- MustNewConstMetric(c.rss, GaugeValue, float64(info.ResidentSize))
90+
ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(info.VirtualSize))
91+
} else {
92+
c.reportError(ch, c.rss, err)
93+
c.reportError(ch, c.vsize, err)
94+
}
8995

9096
if fds, err := getOpenFileCount(); err == nil {
9197
ch <- MustNewConstMetric(c.openFDs, GaugeValue, fds)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
// Copyright 2024 The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package prometheus
15+
16+
import (
17+
"bytes"
18+
"encoding/binary"
19+
"fmt"
20+
"github.com/ebitengine/purego"
21+
)
22+
23+
func init() {
24+
if lib, err := purego.Dlopen("/usr/lib/system/libsystem_kernel.dylib",
25+
purego.RTLD_NOW|purego.RTLD_GLOBAL); err == nil {
26+
27+
// purego.RegisterLibFunc() panics if the symbol is missing. Ignore any error,
28+
// and the metric will simply be unavailable when it is queried, instead of
29+
// bringing down the whole process.
30+
defer func() {
31+
if err := recover(); err != nil {
32+
// TODO: Log this somehow
33+
}
34+
}()
35+
36+
purego.RegisterLibFunc(&machTaskSelf, lib, "mach_task_self")
37+
purego.RegisterLibFunc(&taskInfo, lib, "task_info")
38+
purego.RegisterLibFunc(&machVmRegion, lib, "mach_vm_region")
39+
}
40+
}
41+
42+
const (
43+
// The task_info() flavor MACH_TASK_BASIC_INFO for retrieving machTaskBasicInfo.
44+
mach_task_basic_info task_flavor_t = 20 /* always 64-bit basic info */
45+
46+
// The MACH_TASK_BASIC_INFO_COUNT value, which is passed to the Mach API as the size
47+
// of the payload for MACH_TASK_BASIC_INFO commands.
48+
machTaskBasicInfoCount mach_msg_type_number_t = machTaskBasicInfoSizeOf / 4
49+
)
50+
51+
// Defined in xnu/osfmk/mach/policy.h, xnu/iokit/IOKit/IORPC.h, and xnu/osfmk/mach/message.h
52+
// respectively. policy_t is explicitly 32-bit here to help decoding into a structure,
53+
// where 'int' types are not supported.
54+
type (
55+
policy_t = int32 // typedef int policy_t;
56+
mach_port_t = natural_t // typedef natural_t mach_port_t;
57+
mach_msg_type_number_t = natural_t // typedef natural_t mach_msg_type_number_t;
58+
)
59+
60+
// Defined in xnu/osfmk/mach/task_info.h. Note that task_info_t is actually defined as
61+
// integer_t* and cast from the address of the structure in C. Define it as []byte to
62+
// keep the type system happy.
63+
type (
64+
task_flavor_t = natural_t // typedef natural_t task_flavor_t
65+
task_info_t = []byte // typedef integer_t *task_info_t /* varying array of int */
66+
)
67+
68+
// time_value_t is the type for kernel time values, defined in xnu/osfmk/mach/time_value.h
69+
type time_value_t struct {
70+
Seconds integer_t
71+
MicroSeconds integer_t
72+
}
73+
74+
var machTaskSelf func() mach_port_t
75+
76+
var taskInfo func(
77+
mach_port_t,
78+
task_flavor_t,
79+
task_info_t,
80+
*mach_msg_type_number_t,
81+
) kern_return_t
82+
83+
// machTaskBasicInfo is the representation of `struct mach_task_basic_info` defined in
84+
// xnu/osfmk/mach/task_info.h, which is the architecture independent payload for fetching
85+
// certain task values.
86+
type machTaskBasicInfo struct {
87+
VirtualSize mach_vm_size_t // virtual memory size (bytes)
88+
ResidentSize mach_vm_size_t // resident memory size (bytes)
89+
ResidentSizeMax mach_vm_size_t // maximum resident memory size (bytes)
90+
UserTime time_value_t // total user run time for terminated threads
91+
SystemTime time_value_t // total system run time for terminated threads
92+
Policy policy_t // default policy for new threads
93+
SuspendCount integer_t // suspend count for task
94+
}
95+
96+
func getBasicTaskInfo() (*machTaskBasicInfo, error) {
97+
var info machTaskBasicInfo
98+
99+
if taskInfo == nil {
100+
return nil, fmt.Errorf("task_info() is not supported")
101+
}
102+
103+
var count = machTaskBasicInfoCount
104+
buf := make([]byte, machTaskBasicInfoSizeOf)
105+
106+
if ret := taskInfo(machTaskSelf(), mach_task_basic_info, buf, &count); ret != 0 {
107+
return nil, fmt.Errorf("task_info() returned %d", ret)
108+
}
109+
110+
if err := loadStruct(buf, &info); err != nil {
111+
return nil, err
112+
}
113+
114+
return &info, nil
115+
}
116+
117+
// Defined in xnu/osfmk/mach/memory_object_types.h, xnu/osfmk/mach/vm_behavior.h,
118+
// defined in xnu/osfmk/mach/vm_inherit.h, and defined in xnu/osfmk/mach/vm_prot.h
119+
// respectively. These types are not identical to the native definitions, because the
120+
// struct decoding requires primitives with a specific width. The widths here are the
121+
// same as the native types.
122+
type (
123+
memory_object_offset_t = uint64 // typedef unsigned long long memory_object_offset_t;
124+
vm_behavior_t = int32 // typedef int vm_behavior_t
125+
vm_inherit_t = uint32 // typedef unsigned int vm_inherit_t;
126+
vm_prot_t = int32 // typedef int vm_prot_t;
127+
)
128+
129+
// These are defined in xnu/osfmk/mach/vm_types.h.
130+
type (
131+
vm_map_t = mach_port_t // typedef mach_port_t vm_map_t;
132+
)
133+
134+
// Defined in xnu/osfmk/mach/vm_region.h. vm_region_flavor_t is explicitly 32-bit here to
135+
// help decoding into a structure, where 'int' types are not supported, and vm_region_info_t
136+
// is []byte to keep the type system happy.
137+
type (
138+
vm_region_flavor_t = int32 // typedef int vm_region_flavor_t;
139+
vm_region_info_t = []byte // typedef int *vm_region_info_t;
140+
)
141+
142+
const (
143+
// The mach_vm_region() flavor VM_REGION_BASIC_INFO_64 for retrieving
144+
// vmRegionBasicInfo64.
145+
vm_region_basic_info_64 vm_region_flavor_t = 9
146+
147+
// The VM_REGION_BASIC_INFO_COUNT_64 value, which is passed to the Mach API as the
148+
// size of the payload for VM_REGION_BASIC_INFO_64 commands.
149+
vmRegionBasicInfoCount64 mach_msg_type_number_t = vmRegionBasicInfo64SizeOf / 4
150+
)
151+
152+
// vmRegionBasicInfo64 is the representation of `struct vm_region_basic_info_64` defined
153+
// in xnu/osfmk/mach/vm_region.h. This is enclosed in `#pragma pack(push, 4)` in C, so
154+
// unsafe.SizeOf() won't match the actual sizeof(vm_region_basic_info_64).
155+
type vmRegionBasicInfo64 struct {
156+
Protection vm_prot_t
157+
MaxProtection vm_prot_t
158+
Inheritance vm_inherit_t
159+
Shared boolean_t
160+
Reserved boolean_t
161+
Offset memory_object_offset_t
162+
Behavior vm_behavior_t
163+
UserWiredCount uint16
164+
}
165+
166+
var machVmRegion func(
167+
vm_map_t,
168+
*mach_vm_offset_t, /* IN/OUT */
169+
*mach_vm_size_t, /* OUT */
170+
vm_region_flavor_t, /* IN */
171+
vm_region_info_t, /* OUT */
172+
*mach_msg_type_number_t, /* IN/OUT */
173+
*mach_port_t, /* OUT */
174+
) kern_return_t
175+
176+
func getMemoryUsage() (*machTaskBasicInfo, error) {
177+
// The logic in here follows how the ps(1) utility determines the memory values. The
178+
// basic_task_info command used here is a more modern, cross-architecture one that is
179+
// suggested in the kernel header files.
180+
//
181+
// https://github.com/apple-oss-distributions/adv_cmds/blob/8744084ea0ff41ca4bb96b0f9c22407d0e48e9b7/ps/tasks.c#L132
182+
183+
info, err := getBasicTaskInfo()
184+
185+
if err != nil {
186+
return nil, err
187+
} else if machVmRegion != nil {
188+
189+
var textInfo vmRegionBasicInfo64
190+
191+
buf := make([]byte, vmRegionBasicInfo64SizeOf)
192+
address := globalSharedTextSegment
193+
var size mach_vm_size_t
194+
var objectName mach_port_t
195+
196+
cmd := vm_region_basic_info_64
197+
count := vmRegionBasicInfoCount64
198+
199+
ret := machVmRegion(machTaskSelf(), &address, &size, cmd, buf, &count, &objectName)
200+
201+
if ret == 0 {
202+
if err := loadStruct(buf, &textInfo); err == nil {
203+
adjustment := sharedTextRegionSize + sharedDataRegionSize
204+
if textInfo.Reserved != 0 && size == sharedTextRegionSize && info.VirtualSize > adjustment {
205+
info.VirtualSize -= adjustment
206+
}
207+
}
208+
}
209+
}
210+
211+
return info, nil
212+
}
213+
214+
func loadStruct(buffer []byte, data any) error {
215+
r := bytes.NewReader(buffer)
216+
217+
// TODO: NativeEndian was added in go 1.21
218+
return binary.Read(r, binary.LittleEndian, data)
219+
}

0 commit comments

Comments
 (0)