Skip to content

Commit 3ab224b

Browse files
Hideo Aokidavem330
Hideo Aoki
authored and
davem330
committed
[NET] CORE: Introducing new memory accounting interface.
This patch introduces new memory accounting functions for each network protocol. Most of them are renamed from memory accounting functions for stream protocols. At the same time, some stream memory accounting functions are removed since other functions do same thing. Renaming: sk_stream_free_skb() -> sk_wmem_free_skb() __sk_stream_mem_reclaim() -> __sk_mem_reclaim() sk_stream_mem_reclaim() -> sk_mem_reclaim() sk_stream_mem_schedule -> __sk_mem_schedule() sk_stream_pages() -> sk_mem_pages() sk_stream_rmem_schedule() -> sk_rmem_schedule() sk_stream_wmem_schedule() -> sk_wmem_schedule() sk_charge_skb() -> sk_mem_charge() Removeing sk_stream_rfree(): consolidates into sock_rfree() sk_stream_set_owner_r(): consolidates into skb_set_owner_r() sk_stream_mem_schedule() The following functions are added. sk_has_account(): check if the protocol supports accounting sk_mem_uncharge(): do the opposite of sk_mem_charge() In addition, to achieve consolidation, updating sk_wmem_queued is removed from sk_mem_charge(). Next, to consolidate memory accounting functions, this patch adds memory accounting calls to network core functions. Moreover, present memory accounting call is renamed to new accounting call. Finally we replace present memory accounting calls with new interface in TCP and SCTP. Signed-off-by: Takahiro Yasui <[email protected]> Signed-off-by: Hideo Aoki <[email protected]> Signed-off-by: David S. Miller <[email protected]>
1 parent a06b494 commit 3ab224b

File tree

15 files changed

+222
-175
lines changed

15 files changed

+222
-175
lines changed

‎include/net/sctp/sctp.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -463,8 +463,7 @@ static inline void sctp_skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
463463
skb->destructor = sctp_sock_rfree;
464464
atomic_add(event->rmem_len, &sk->sk_rmem_alloc);
465465
/*
466-
* This mimics the behavior of
467-
* sk_stream_set_owner_r
466+
* This mimics the behavior of skb_set_owner_r
468467
*/
469468
sk->sk_forward_alloc -= event->rmem_len;
470469
}

‎include/net/sock.h

Lines changed: 57 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -460,25 +460,6 @@ static inline int sk_stream_memory_free(struct sock *sk)
460460
return sk->sk_wmem_queued < sk->sk_sndbuf;
461461
}
462462

463-
extern void sk_stream_rfree(struct sk_buff *skb);
464-
465-
static inline void sk_stream_set_owner_r(struct sk_buff *skb, struct sock *sk)
466-
{
467-
skb->sk = sk;
468-
skb->destructor = sk_stream_rfree;
469-
atomic_add(skb->truesize, &sk->sk_rmem_alloc);
470-
sk->sk_forward_alloc -= skb->truesize;
471-
}
472-
473-
static inline void sk_stream_free_skb(struct sock *sk, struct sk_buff *skb)
474-
{
475-
skb_truesize_check(skb);
476-
sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
477-
sk->sk_wmem_queued -= skb->truesize;
478-
sk->sk_forward_alloc += skb->truesize;
479-
__kfree_skb(skb);
480-
}
481-
482463
/* The per-socket spinlock must be held here. */
483464
static inline void sk_add_backlog(struct sock *sk, struct sk_buff *skb)
484465
{
@@ -576,7 +557,7 @@ struct proto {
576557
/*
577558
* Pressure flag: try to collapse.
578559
* Technical note: it is used by multiple contexts non atomically.
579-
* All the sk_stream_mem_schedule() is of this nature: accounting
560+
* All the __sk_mem_schedule() is of this nature: accounting
580561
* is strict, actions are advisory and have some latency.
581562
*/
582563
int *memory_pressure;
@@ -712,33 +693,73 @@ static inline struct inode *SOCK_INODE(struct socket *socket)
712693
return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
713694
}
714695

715-
extern void __sk_stream_mem_reclaim(struct sock *sk);
716-
extern int sk_stream_mem_schedule(struct sock *sk, int size, int kind);
696+
/*
697+
* Functions for memory accounting
698+
*/
699+
extern int __sk_mem_schedule(struct sock *sk, int size, int kind);
700+
extern void __sk_mem_reclaim(struct sock *sk);
717701

718-
#define SK_STREAM_MEM_QUANTUM ((int)PAGE_SIZE)
719-
#define SK_STREAM_MEM_QUANTUM_SHIFT ilog2(SK_STREAM_MEM_QUANTUM)
702+
#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
703+
#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
704+
#define SK_MEM_SEND 0
705+
#define SK_MEM_RECV 1
720706

721-
static inline int sk_stream_pages(int amt)
707+
static inline int sk_mem_pages(int amt)
722708
{
723-
return (amt + SK_STREAM_MEM_QUANTUM - 1) >> SK_STREAM_MEM_QUANTUM_SHIFT;
709+
return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
724710
}
725711

726-
static inline void sk_stream_mem_reclaim(struct sock *sk)
712+
static inline int sk_has_account(struct sock *sk)
727713
{
728-
if (sk->sk_forward_alloc >= SK_STREAM_MEM_QUANTUM)
729-
__sk_stream_mem_reclaim(sk);
714+
/* return true if protocol supports memory accounting */
715+
return !!sk->sk_prot->memory_allocated;
730716
}
731717

732-
static inline int sk_stream_rmem_schedule(struct sock *sk, struct sk_buff *skb)
718+
static inline int sk_wmem_schedule(struct sock *sk, int size)
733719
{
734-
return (int)skb->truesize <= sk->sk_forward_alloc ||
735-
sk_stream_mem_schedule(sk, skb->truesize, 1);
720+
if (!sk_has_account(sk))
721+
return 1;
722+
return size <= sk->sk_forward_alloc ||
723+
__sk_mem_schedule(sk, size, SK_MEM_SEND);
736724
}
737725

738-
static inline int sk_stream_wmem_schedule(struct sock *sk, int size)
726+
static inline int sk_rmem_schedule(struct sock *sk, int size)
739727
{
728+
if (!sk_has_account(sk))
729+
return 1;
740730
return size <= sk->sk_forward_alloc ||
741-
sk_stream_mem_schedule(sk, size, 0);
731+
__sk_mem_schedule(sk, size, SK_MEM_RECV);
732+
}
733+
734+
static inline void sk_mem_reclaim(struct sock *sk)
735+
{
736+
if (!sk_has_account(sk))
737+
return;
738+
if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
739+
__sk_mem_reclaim(sk);
740+
}
741+
742+
static inline void sk_mem_charge(struct sock *sk, int size)
743+
{
744+
if (!sk_has_account(sk))
745+
return;
746+
sk->sk_forward_alloc -= size;
747+
}
748+
749+
static inline void sk_mem_uncharge(struct sock *sk, int size)
750+
{
751+
if (!sk_has_account(sk))
752+
return;
753+
sk->sk_forward_alloc += size;
754+
}
755+
756+
static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
757+
{
758+
skb_truesize_check(skb);
759+
sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
760+
sk->sk_wmem_queued -= skb->truesize;
761+
sk_mem_uncharge(sk, skb->truesize);
762+
__kfree_skb(skb);
742763
}
743764

744765
/* Used by processes to "lock" a socket state, so that
@@ -1076,12 +1097,6 @@ static inline int sk_can_gso(const struct sock *sk)
10761097

10771098
extern void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
10781099

1079-
static inline void sk_charge_skb(struct sock *sk, struct sk_buff *skb)
1080-
{
1081-
sk->sk_wmem_queued += skb->truesize;
1082-
sk->sk_forward_alloc -= skb->truesize;
1083-
}
1084-
10851100
static inline int skb_copy_to_page(struct sock *sk, char __user *from,
10861101
struct sk_buff *skb, struct page *page,
10871102
int off, int copy)
@@ -1101,7 +1116,7 @@ static inline int skb_copy_to_page(struct sock *sk, char __user *from,
11011116
skb->data_len += copy;
11021117
skb->truesize += copy;
11031118
sk->sk_wmem_queued += copy;
1104-
sk->sk_forward_alloc -= copy;
1119+
sk_mem_charge(sk, copy);
11051120
return 0;
11061121
}
11071122

@@ -1127,6 +1142,7 @@ static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
11271142
skb->sk = sk;
11281143
skb->destructor = sock_rfree;
11291144
atomic_add(skb->truesize, &sk->sk_rmem_alloc);
1145+
sk_mem_charge(sk, skb->truesize);
11301146
}
11311147

11321148
extern void sk_reset_timer(struct sock *sk, struct timer_list* timer,

‎include/net/tcp.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,8 +1196,8 @@ static inline void tcp_write_queue_purge(struct sock *sk)
11961196
struct sk_buff *skb;
11971197

11981198
while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
1199-
sk_stream_free_skb(sk, skb);
1200-
sk_stream_mem_reclaim(sk);
1199+
sk_wmem_free_skb(sk, skb);
1200+
sk_mem_reclaim(sk);
12011201
}
12021202

12031203
static inline struct sk_buff *tcp_write_queue_head(struct sock *sk)

‎net/core/datagram.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags,
209209
void skb_free_datagram(struct sock *sk, struct sk_buff *skb)
210210
{
211211
kfree_skb(skb);
212+
sk_mem_reclaim(sk);
212213
}
213214

214215
/**
@@ -248,6 +249,7 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
248249
}
249250

250251
kfree_skb(skb);
252+
sk_mem_reclaim(sk);
251253
return err;
252254
}
253255

‎net/core/sock.c

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,11 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
282282
if (err)
283283
goto out;
284284

285+
if (!sk_rmem_schedule(sk, skb->truesize)) {
286+
err = -ENOBUFS;
287+
goto out;
288+
}
289+
285290
skb->dev = NULL;
286291
skb_set_owner_r(skb, sk);
287292

@@ -1107,7 +1112,9 @@ void sock_rfree(struct sk_buff *skb)
11071112
{
11081113
struct sock *sk = skb->sk;
11091114

1115+
skb_truesize_check(skb);
11101116
atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
1117+
sk_mem_uncharge(skb->sk, skb->truesize);
11111118
}
11121119

11131120

@@ -1384,6 +1391,103 @@ int sk_wait_data(struct sock *sk, long *timeo)
13841391

13851392
EXPORT_SYMBOL(sk_wait_data);
13861393

1394+
/**
1395+
* __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
1396+
* @sk: socket
1397+
* @size: memory size to allocate
1398+
* @kind: allocation type
1399+
*
1400+
* If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
1401+
* rmem allocation. This function assumes that protocols which have
1402+
* memory_pressure use sk_wmem_queued as write buffer accounting.
1403+
*/
1404+
int __sk_mem_schedule(struct sock *sk, int size, int kind)
1405+
{
1406+
struct proto *prot = sk->sk_prot;
1407+
int amt = sk_mem_pages(size);
1408+
int allocated;
1409+
1410+
sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
1411+
allocated = atomic_add_return(amt, prot->memory_allocated);
1412+
1413+
/* Under limit. */
1414+
if (allocated <= prot->sysctl_mem[0]) {
1415+
if (prot->memory_pressure && *prot->memory_pressure)
1416+
*prot->memory_pressure = 0;
1417+
return 1;
1418+
}
1419+
1420+
/* Under pressure. */
1421+
if (allocated > prot->sysctl_mem[1])
1422+
if (prot->enter_memory_pressure)
1423+
prot->enter_memory_pressure();
1424+
1425+
/* Over hard limit. */
1426+
if (allocated > prot->sysctl_mem[2])
1427+
goto suppress_allocation;
1428+
1429+
/* guarantee minimum buffer size under pressure */
1430+
if (kind == SK_MEM_RECV) {
1431+
if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
1432+
return 1;
1433+
} else { /* SK_MEM_SEND */
1434+
if (sk->sk_type == SOCK_STREAM) {
1435+
if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
1436+
return 1;
1437+
} else if (atomic_read(&sk->sk_wmem_alloc) <
1438+
prot->sysctl_wmem[0])
1439+
return 1;
1440+
}
1441+
1442+
if (prot->memory_pressure) {
1443+
if (!*prot->memory_pressure ||
1444+
prot->sysctl_mem[2] > atomic_read(prot->sockets_allocated) *
1445+
sk_mem_pages(sk->sk_wmem_queued +
1446+
atomic_read(&sk->sk_rmem_alloc) +
1447+
sk->sk_forward_alloc))
1448+
return 1;
1449+
}
1450+
1451+
suppress_allocation:
1452+
1453+
if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
1454+
sk_stream_moderate_sndbuf(sk);
1455+
1456+
/* Fail only if socket is _under_ its sndbuf.
1457+
* In this case we cannot block, so that we have to fail.
1458+
*/
1459+
if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
1460+
return 1;
1461+
}
1462+
1463+
/* Alas. Undo changes. */
1464+
sk->sk_forward_alloc -= amt * SK_MEM_QUANTUM;
1465+
atomic_sub(amt, prot->memory_allocated);
1466+
return 0;
1467+
}
1468+
1469+
EXPORT_SYMBOL(__sk_mem_schedule);
1470+
1471+
/**
1472+
* __sk_reclaim - reclaim memory_allocated
1473+
* @sk: socket
1474+
*/
1475+
void __sk_mem_reclaim(struct sock *sk)
1476+
{
1477+
struct proto *prot = sk->sk_prot;
1478+
1479+
atomic_sub(sk->sk_forward_alloc / SK_MEM_QUANTUM,
1480+
prot->memory_allocated);
1481+
sk->sk_forward_alloc &= SK_MEM_QUANTUM - 1;
1482+
1483+
if (prot->memory_pressure && *prot->memory_pressure &&
1484+
(atomic_read(prot->memory_allocated) < prot->sysctl_mem[0]))
1485+
*prot->memory_pressure = 0;
1486+
}
1487+
1488+
EXPORT_SYMBOL(__sk_mem_reclaim);
1489+
1490+
13871491
/*
13881492
* Set of default routines for initialising struct proto_ops when
13891493
* the protocol does not support a particular function. In certain

0 commit comments

Comments
 (0)