Skip to content

Commit b90a126

Browse files
davem330davem330
committed
Merge branch 'net-openvswitch-masks-cache-enhancements'
Eelco Chaudron says: ==================== net: openvswitch: masks cache enhancements This patchset adds two enhancements to the Open vSwitch masks cache. Changes in v4 [patch 2/2 only]: - Remove null check before calling free_percpu() - Make ovs_dp_change() return appropriate error codes Changes in v3 [patch 2/2 only]: - Use is_power_of_2() function - Use array_size() function - Fix remaining sparse errors Changes in v2 [patch 2/2 only]: - Fix sparse warnings - Fix netlink policy items reported by Florian Westphal ==================== Signed-off-by: Eelco Chaudron <[email protected]> Signed-off-by: David S. Miller <[email protected]>
2 parents d652692 + 9bf24f5 commit b90a126

File tree

5 files changed

+139
-22
lines changed

5 files changed

+139
-22
lines changed

‎include/uapi/linux/openvswitch.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ enum ovs_datapath_attr {
8686
OVS_DP_ATTR_MEGAFLOW_STATS, /* struct ovs_dp_megaflow_stats */
8787
OVS_DP_ATTR_USER_FEATURES, /* OVS_DP_F_* */
8888
OVS_DP_ATTR_PAD,
89+
OVS_DP_ATTR_MASKS_CACHE_SIZE,
8990
__OVS_DP_ATTR_MAX
9091
};
9192

@@ -102,8 +103,8 @@ struct ovs_dp_megaflow_stats {
102103
__u64 n_mask_hit; /* Number of masks used for flow lookups. */
103104
__u32 n_masks; /* Number of masks for the datapath. */
104105
__u32 pad0; /* Pad for future expension. */
106+
__u64 n_cache_hit; /* Number of cache matches for flow lookups. */
105107
__u64 pad1; /* Pad for future expension. */
106-
__u64 pad2; /* Pad for future expension. */
107108
};
108109

109110
struct ovs_vport_stats {

‎net/openvswitch/datapath.c

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,13 +225,14 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
225225
struct dp_stats_percpu *stats;
226226
u64 *stats_counter;
227227
u32 n_mask_hit;
228+
u32 n_cache_hit;
228229
int error;
229230

230231
stats = this_cpu_ptr(dp->stats_percpu);
231232

232233
/* Look up flow. */
233234
flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
234-
&n_mask_hit);
235+
&n_mask_hit, &n_cache_hit);
235236
if (unlikely(!flow)) {
236237
struct dp_upcall_info upcall;
237238

@@ -262,6 +263,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
262263
u64_stats_update_begin(&stats->syncp);
263264
(*stats_counter)++;
264265
stats->n_mask_hit += n_mask_hit;
266+
stats->n_cache_hit += n_cache_hit;
265267
u64_stats_update_end(&stats->syncp);
266268
}
267269

@@ -699,6 +701,7 @@ static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
699701
stats->n_missed += local_stats.n_missed;
700702
stats->n_lost += local_stats.n_lost;
701703
mega_stats->n_mask_hit += local_stats.n_mask_hit;
704+
mega_stats->n_cache_hit += local_stats.n_cache_hit;
702705
}
703706
}
704707

@@ -1495,6 +1498,7 @@ static size_t ovs_dp_cmd_msg_size(void)
14951498
msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
14961499
msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
14971500
msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1501+
msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
14981502

14991503
return msgsize;
15001504
}
@@ -1532,6 +1536,10 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
15321536
if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
15331537
goto nla_put_failure;
15341538

1539+
if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE,
1540+
ovs_flow_tbl_masks_cache_size(&dp->table)))
1541+
goto nla_put_failure;
1542+
15351543
genlmsg_end(skb, ovs_header);
15361544
return 0;
15371545

@@ -1596,6 +1604,16 @@ static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])
15961604
#endif
15971605
}
15981606

1607+
if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
1608+
int err;
1609+
u32 cache_size;
1610+
1611+
cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
1612+
err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size);
1613+
if (err)
1614+
return err;
1615+
}
1616+
15991617
dp->user_features = user_features;
16001618

16011619
if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING)
@@ -1884,6 +1902,8 @@ static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
18841902
[OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
18851903
[OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
18861904
[OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1905+
[OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0,
1906+
PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
18871907
};
18881908

18891909
static const struct genl_ops dp_datapath_genl_ops[] = {

‎net/openvswitch/datapath.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,15 @@
3838
* @n_mask_hit: Number of masks looked up for flow match.
3939
* @n_mask_hit / (@n_hit + @n_missed) will be the average masks looked
4040
* up per packet.
41+
* @n_cache_hit: The number of received packets that had their mask found using
42+
* the mask cache.
4143
*/
4244
struct dp_stats_percpu {
4345
u64 n_hit;
4446
u64 n_missed;
4547
u64 n_lost;
4648
u64 n_mask_hit;
49+
u64 n_cache_hit;
4750
struct u64_stats_sync syncp;
4851
};
4952

‎net/openvswitch/flow_table.c

Lines changed: 102 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@
3838
#define MASK_ARRAY_SIZE_MIN 16
3939
#define REHASH_INTERVAL (10 * 60 * HZ)
4040

41+
#define MC_DEFAULT_HASH_ENTRIES 256
4142
#define MC_HASH_SHIFT 8
42-
#define MC_HASH_ENTRIES (1u << MC_HASH_SHIFT)
4343
#define MC_HASH_SEGS ((sizeof(uint32_t) * 8) / MC_HASH_SHIFT)
4444

4545
static struct kmem_cache *flow_cache;
@@ -341,15 +341,79 @@ static void flow_mask_remove(struct flow_table *tbl, struct sw_flow_mask *mask)
341341
}
342342
}
343343

344+
static void __mask_cache_destroy(struct mask_cache *mc)
345+
{
346+
free_percpu(mc->mask_cache);
347+
kfree(mc);
348+
}
349+
350+
static void mask_cache_rcu_cb(struct rcu_head *rcu)
351+
{
352+
struct mask_cache *mc = container_of(rcu, struct mask_cache, rcu);
353+
354+
__mask_cache_destroy(mc);
355+
}
356+
357+
static struct mask_cache *tbl_mask_cache_alloc(u32 size)
358+
{
359+
struct mask_cache_entry __percpu *cache = NULL;
360+
struct mask_cache *new;
361+
362+
/* Only allow size to be 0, or a power of 2, and does not exceed
363+
* percpu allocation size.
364+
*/
365+
if ((!is_power_of_2(size) && size != 0) ||
366+
(size * sizeof(struct mask_cache_entry)) > PCPU_MIN_UNIT_SIZE)
367+
return NULL;
368+
369+
new = kzalloc(sizeof(*new), GFP_KERNEL);
370+
if (!new)
371+
return NULL;
372+
373+
new->cache_size = size;
374+
if (new->cache_size > 0) {
375+
cache = __alloc_percpu(array_size(sizeof(struct mask_cache_entry),
376+
new->cache_size),
377+
__alignof__(struct mask_cache_entry));
378+
if (!cache) {
379+
kfree(new);
380+
return NULL;
381+
}
382+
}
383+
384+
new->mask_cache = cache;
385+
return new;
386+
}
387+
int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size)
388+
{
389+
struct mask_cache *mc = rcu_dereference(table->mask_cache);
390+
struct mask_cache *new;
391+
392+
if (size == mc->cache_size)
393+
return 0;
394+
395+
if ((!is_power_of_2(size) && size != 0) ||
396+
(size * sizeof(struct mask_cache_entry)) > PCPU_MIN_UNIT_SIZE)
397+
return -EINVAL;
398+
399+
new = tbl_mask_cache_alloc(size);
400+
if (!new)
401+
return -ENOMEM;
402+
403+
rcu_assign_pointer(table->mask_cache, new);
404+
call_rcu(&mc->rcu, mask_cache_rcu_cb);
405+
406+
return 0;
407+
}
408+
344409
int ovs_flow_tbl_init(struct flow_table *table)
345410
{
346411
struct table_instance *ti, *ufid_ti;
412+
struct mask_cache *mc;
347413
struct mask_array *ma;
348414

349-
table->mask_cache = __alloc_percpu(sizeof(struct mask_cache_entry) *
350-
MC_HASH_ENTRIES,
351-
__alignof__(struct mask_cache_entry));
352-
if (!table->mask_cache)
415+
mc = tbl_mask_cache_alloc(MC_DEFAULT_HASH_ENTRIES);
416+
if (!mc)
353417
return -ENOMEM;
354418

355419
ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);
@@ -367,6 +431,7 @@ int ovs_flow_tbl_init(struct flow_table *table)
367431
rcu_assign_pointer(table->ti, ti);
368432
rcu_assign_pointer(table->ufid_ti, ufid_ti);
369433
rcu_assign_pointer(table->mask_array, ma);
434+
rcu_assign_pointer(table->mask_cache, mc);
370435
table->last_rehash = jiffies;
371436
table->count = 0;
372437
table->ufid_count = 0;
@@ -377,7 +442,7 @@ int ovs_flow_tbl_init(struct flow_table *table)
377442
free_mask_array:
378443
__mask_array_destroy(ma);
379444
free_mask_cache:
380-
free_percpu(table->mask_cache);
445+
__mask_cache_destroy(mc);
381446
return -ENOMEM;
382447
}
383448

@@ -453,9 +518,11 @@ void ovs_flow_tbl_destroy(struct flow_table *table)
453518
{
454519
struct table_instance *ti = rcu_dereference_raw(table->ti);
455520
struct table_instance *ufid_ti = rcu_dereference_raw(table->ufid_ti);
521+
struct mask_cache *mc = rcu_dereference(table->mask_cache);
522+
struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);
456523

457-
free_percpu(table->mask_cache);
458-
call_rcu(&table->mask_array->rcu, mask_array_rcu_cb);
524+
call_rcu(&mc->rcu, mask_cache_rcu_cb);
525+
call_rcu(&ma->rcu, mask_array_rcu_cb);
459526
table_instance_destroy(table, ti, ufid_ti, false);
460527
}
461528

@@ -667,6 +734,7 @@ static struct sw_flow *flow_lookup(struct flow_table *tbl,
667734
struct mask_array *ma,
668735
const struct sw_flow_key *key,
669736
u32 *n_mask_hit,
737+
u32 *n_cache_hit,
670738
u32 *index)
671739
{
672740
u64 *usage_counters = this_cpu_ptr(ma->masks_usage_cntr);
@@ -682,6 +750,7 @@ static struct sw_flow *flow_lookup(struct flow_table *tbl,
682750
u64_stats_update_begin(&ma->syncp);
683751
usage_counters[*index]++;
684752
u64_stats_update_end(&ma->syncp);
753+
(*n_cache_hit)++;
685754
return flow;
686755
}
687756
}
@@ -719,8 +788,10 @@ static struct sw_flow *flow_lookup(struct flow_table *tbl,
719788
struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl,
720789
const struct sw_flow_key *key,
721790
u32 skb_hash,
722-
u32 *n_mask_hit)
791+
u32 *n_mask_hit,
792+
u32 *n_cache_hit)
723793
{
794+
struct mask_cache *mc = rcu_dereference(tbl->mask_cache);
724795
struct mask_array *ma = rcu_dereference(tbl->mask_array);
725796
struct table_instance *ti = rcu_dereference(tbl->ti);
726797
struct mask_cache_entry *entries, *ce;
@@ -729,10 +800,13 @@ struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl,
729800
int seg;
730801

731802
*n_mask_hit = 0;
732-
if (unlikely(!skb_hash)) {
803+
*n_cache_hit = 0;
804+
if (unlikely(!skb_hash || mc->cache_size == 0)) {
733805
u32 mask_index = 0;
806+
u32 cache = 0;
734807

735-
return flow_lookup(tbl, ti, ma, key, n_mask_hit, &mask_index);
808+
return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,
809+
&mask_index);
736810
}
737811

738812
/* Pre and post recirulation flows usually have the same skb_hash
@@ -743,17 +817,17 @@ struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl,
743817

744818
ce = NULL;
745819
hash = skb_hash;
746-
entries = this_cpu_ptr(tbl->mask_cache);
820+
entries = this_cpu_ptr(mc->mask_cache);
747821

748822
/* Find the cache entry 'ce' to operate on. */
749823
for (seg = 0; seg < MC_HASH_SEGS; seg++) {
750-
int index = hash & (MC_HASH_ENTRIES - 1);
824+
int index = hash & (mc->cache_size - 1);
751825
struct mask_cache_entry *e;
752826

753827
e = &entries[index];
754828
if (e->skb_hash == skb_hash) {
755829
flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,
756-
&e->mask_index);
830+
n_cache_hit, &e->mask_index);
757831
if (!flow)
758832
e->skb_hash = 0;
759833
return flow;
@@ -766,10 +840,12 @@ struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl,
766840
}
767841

768842
/* Cache miss, do full lookup. */
769-
flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, &ce->mask_index);
843+
flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,
844+
&ce->mask_index);
770845
if (flow)
771846
ce->skb_hash = skb_hash;
772847

848+
*n_cache_hit = 0;
773849
return flow;
774850
}
775851

@@ -779,9 +855,10 @@ struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *tbl,
779855
struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);
780856
struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);
781857
u32 __always_unused n_mask_hit;
858+
u32 __always_unused n_cache_hit;
782859
u32 index = 0;
783860

784-
return flow_lookup(tbl, ti, ma, key, &n_mask_hit, &index);
861+
return flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);
785862
}
786863

787864
struct sw_flow *ovs_flow_tbl_lookup_exact(struct flow_table *tbl,
@@ -858,6 +935,13 @@ int ovs_flow_tbl_num_masks(const struct flow_table *table)
858935
return READ_ONCE(ma->count);
859936
}
860937

938+
u32 ovs_flow_tbl_masks_cache_size(const struct flow_table *table)
939+
{
940+
struct mask_cache *mc = rcu_dereference(table->mask_cache);
941+
942+
return READ_ONCE(mc->cache_size);
943+
}
944+
861945
static struct table_instance *table_instance_expand(struct table_instance *ti,
862946
bool ufid)
863947
{
@@ -1086,8 +1170,8 @@ void ovs_flow_masks_rebalance(struct flow_table *table)
10861170
for (i = 0; i < masks_entries; i++) {
10871171
int index = masks_and_count[i].index;
10881172

1089-
new->masks[new->count++] =
1090-
rcu_dereference_ovsl(ma->masks[index]);
1173+
if (ovsl_dereference(ma->masks[index]))
1174+
new->masks[new->count++] = ma->masks[index];
10911175
}
10921176

10931177
rcu_assign_pointer(table->mask_array, new);

‎net/openvswitch/flow_table.h

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ struct mask_cache_entry {
2727
u32 mask_index;
2828
};
2929

30+
struct mask_cache {
31+
struct rcu_head rcu;
32+
u32 cache_size; /* Must be ^2 value. */
33+
struct mask_cache_entry __percpu *mask_cache;
34+
};
35+
3036
struct mask_count {
3137
int index;
3238
u64 counter;
@@ -53,7 +59,7 @@ struct table_instance {
5359
struct flow_table {
5460
struct table_instance __rcu *ti;
5561
struct table_instance __rcu *ufid_ti;
56-
struct mask_cache_entry __percpu *mask_cache;
62+
struct mask_cache __rcu *mask_cache;
5763
struct mask_array __rcu *mask_array;
5864
unsigned long last_rehash;
5965
unsigned int count;
@@ -77,12 +83,15 @@ int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,
7783
const struct sw_flow_mask *mask);
7884
void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow);
7985
int ovs_flow_tbl_num_masks(const struct flow_table *table);
86+
u32 ovs_flow_tbl_masks_cache_size(const struct flow_table *table);
87+
int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size);
8088
struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *table,
8189
u32 *bucket, u32 *idx);
8290
struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *,
8391
const struct sw_flow_key *,
8492
u32 skb_hash,
85-
u32 *n_mask_hit);
93+
u32 *n_mask_hit,
94+
u32 *n_cache_hit);
8695
struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *,
8796
const struct sw_flow_key *);
8897
struct sw_flow *ovs_flow_tbl_lookup_exact(struct flow_table *tbl,

0 commit comments

Comments
 (0)