Skip to content

Commit 450af8d

Browse files
sinkapAlexei Starovoitov
sinkap
authored and
Alexei Starovoitov
committed
bpf: Split bpf_local_storage to bpf_sk_storage
A purely mechanical change: bpf_sk_storage.c = bpf_sk_storage.c + bpf_local_storage.c bpf_sk_storage.h = bpf_sk_storage.h + bpf_local_storage.h Signed-off-by: KP Singh <[email protected]> Signed-off-by: Alexei Starovoitov <[email protected]> Acked-by: Martin KaFai Lau <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent f836a56 commit 450af8d

File tree

5 files changed

+766
-731
lines changed

5 files changed

+766
-731
lines changed

‎include/linux/bpf_local_storage.h

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* Copyright (c) 2019 Facebook
4+
* Copyright 2020 Google LLC.
5+
*/
6+
7+
#ifndef _BPF_LOCAL_STORAGE_H
8+
#define _BPF_LOCAL_STORAGE_H
9+
10+
#include <linux/bpf.h>
11+
#include <linux/rculist.h>
12+
#include <linux/list.h>
13+
#include <linux/hash.h>
14+
#include <linux/types.h>
15+
#include <uapi/linux/btf.h>
16+
17+
#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
18+
19+
struct bpf_local_storage_map_bucket {
20+
struct hlist_head list;
21+
raw_spinlock_t lock;
22+
};
23+
24+
/* Thp map is not the primary owner of a bpf_local_storage_elem.
25+
* Instead, the container object (eg. sk->sk_bpf_storage) is.
26+
*
27+
* The map (bpf_local_storage_map) is for two purposes
28+
* 1. Define the size of the "local storage". It is
29+
* the map's value_size.
30+
*
31+
* 2. Maintain a list to keep track of all elems such
32+
* that they can be cleaned up during the map destruction.
33+
*
34+
* When a bpf local storage is being looked up for a
35+
* particular object, the "bpf_map" pointer is actually used
36+
* as the "key" to search in the list of elem in
37+
* the respective bpf_local_storage owned by the object.
38+
*
39+
* e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
40+
* as the searching key.
41+
*/
42+
struct bpf_local_storage_map {
43+
struct bpf_map map;
44+
/* Lookup elem does not require accessing the map.
45+
*
46+
* Updating/Deleting requires a bucket lock to
47+
* link/unlink the elem from the map. Having
48+
* multiple buckets to improve contention.
49+
*/
50+
struct bpf_local_storage_map_bucket *buckets;
51+
u32 bucket_log;
52+
u16 elem_size;
53+
u16 cache_idx;
54+
};
55+
56+
struct bpf_local_storage_data {
57+
/* smap is used as the searching key when looking up
58+
* from the object's bpf_local_storage.
59+
*
60+
* Put it in the same cacheline as the data to minimize
61+
* the number of cachelines access during the cache hit case.
62+
*/
63+
struct bpf_local_storage_map __rcu *smap;
64+
u8 data[] __aligned(8);
65+
};
66+
67+
/* Linked to bpf_local_storage and bpf_local_storage_map */
68+
struct bpf_local_storage_elem {
69+
struct hlist_node map_node; /* Linked to bpf_local_storage_map */
70+
struct hlist_node snode; /* Linked to bpf_local_storage */
71+
struct bpf_local_storage __rcu *local_storage;
72+
struct rcu_head rcu;
73+
/* 8 bytes hole */
74+
/* The data is stored in aother cacheline to minimize
75+
* the number of cachelines access during a cache hit.
76+
*/
77+
struct bpf_local_storage_data sdata ____cacheline_aligned;
78+
};
79+
80+
struct bpf_local_storage {
81+
struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
82+
struct hlist_head list; /* List of bpf_local_storage_elem */
83+
void *owner; /* The object that owns the above "list" of
84+
* bpf_local_storage_elem.
85+
*/
86+
struct rcu_head rcu;
87+
raw_spinlock_t lock; /* Protect adding/removing from the "list" */
88+
};
89+
90+
/* U16_MAX is much more than enough for sk local storage
91+
* considering a tcp_sock is ~2k.
92+
*/
93+
#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \
94+
min_t(u32, \
95+
(KMALLOC_MAX_SIZE - MAX_BPF_STACK - \
96+
sizeof(struct bpf_local_storage_elem)), \
97+
(U16_MAX - sizeof(struct bpf_local_storage_elem)))
98+
99+
#define SELEM(_SDATA) \
100+
container_of((_SDATA), struct bpf_local_storage_elem, sdata)
101+
#define SDATA(_SELEM) (&(_SELEM)->sdata)
102+
103+
#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
104+
105+
struct bpf_local_storage_cache {
106+
spinlock_t idx_lock;
107+
u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
108+
};
109+
110+
#define DEFINE_BPF_STORAGE_CACHE(name) \
111+
static struct bpf_local_storage_cache name = { \
112+
.idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \
113+
}
114+
115+
u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
116+
void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
117+
u16 idx);
118+
119+
/* Helper functions for bpf_local_storage */
120+
int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
121+
122+
struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
123+
124+
struct bpf_local_storage_data *
125+
bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
126+
struct bpf_local_storage_map *smap,
127+
bool cacheit_lockit);
128+
129+
void bpf_local_storage_map_free(struct bpf_local_storage_map *smap);
130+
131+
int bpf_local_storage_map_check_btf(const struct bpf_map *map,
132+
const struct btf *btf,
133+
const struct btf_type *key_type,
134+
const struct btf_type *value_type);
135+
136+
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
137+
struct bpf_local_storage_elem *selem);
138+
139+
bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
140+
struct bpf_local_storage_elem *selem,
141+
bool uncharge_omem);
142+
143+
void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
144+
145+
void bpf_selem_link_map(struct bpf_local_storage_map *smap,
146+
struct bpf_local_storage_elem *selem);
147+
148+
void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
149+
150+
struct bpf_local_storage_elem *
151+
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
152+
bool charge_mem);
153+
154+
int
155+
bpf_local_storage_alloc(void *owner,
156+
struct bpf_local_storage_map *smap,
157+
struct bpf_local_storage_elem *first_selem);
158+
159+
struct bpf_local_storage_data *
160+
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
161+
void *value, u64 map_flags);
162+
163+
#endif /* _BPF_LOCAL_STORAGE_H */

‎include/net/bpf_sk_storage.h

+1-60
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <net/sock.h>
1313
#include <uapi/linux/sock_diag.h>
1414
#include <uapi/linux/btf.h>
15+
#include <linux/bpf_local_storage.h>
1516

1617
struct sock;
1718

@@ -26,66 +27,6 @@ struct sk_buff;
2627
struct nlattr;
2728
struct sock;
2829

29-
#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
30-
31-
struct bpf_local_storage_cache {
32-
spinlock_t idx_lock;
33-
u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
34-
};
35-
36-
#define DEFINE_BPF_STORAGE_CACHE(name) \
37-
static struct bpf_local_storage_cache name = { \
38-
.idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \
39-
}
40-
41-
u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
42-
void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
43-
u16 idx);
44-
45-
/* Helper functions for bpf_local_storage */
46-
int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
47-
48-
struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
49-
50-
struct bpf_local_storage_data *
51-
bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
52-
struct bpf_local_storage_map *smap,
53-
bool cacheit_lockit);
54-
55-
void bpf_local_storage_map_free(struct bpf_local_storage_map *smap);
56-
57-
int bpf_local_storage_map_check_btf(const struct bpf_map *map,
58-
const struct btf *btf,
59-
const struct btf_type *key_type,
60-
const struct btf_type *value_type);
61-
62-
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
63-
struct bpf_local_storage_elem *selem);
64-
65-
bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
66-
struct bpf_local_storage_elem *selem,
67-
bool uncharge_omem);
68-
69-
void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
70-
71-
void bpf_selem_link_map(struct bpf_local_storage_map *smap,
72-
struct bpf_local_storage_elem *selem);
73-
74-
void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
75-
76-
struct bpf_local_storage_elem *
77-
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
78-
bool charge_mem);
79-
80-
int
81-
bpf_local_storage_alloc(void *owner,
82-
struct bpf_local_storage_map *smap,
83-
struct bpf_local_storage_elem *first_selem);
84-
85-
struct bpf_local_storage_data *
86-
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
87-
void *value, u64 map_flags);
88-
8930
#ifdef CONFIG_BPF_SYSCALL
9031
int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk);
9132
struct bpf_sk_storage_diag *

‎kernel/bpf/Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ obj-$(CONFIG_BPF_JIT) += dispatcher.o
1212
ifeq ($(CONFIG_NET),y)
1313
obj-$(CONFIG_BPF_SYSCALL) += devmap.o
1414
obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
15+
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o
1516
obj-$(CONFIG_BPF_SYSCALL) += offload.o
1617
obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
1718
endif

0 commit comments

Comments
 (0)