38
38
39
39
/* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */
40
40
#define GUARD_SZ (1ull << sizeof(((struct bpf_insn *)0)->off) * 8)
41
- #define KERN_VM_SZ ((1ull << 32) + GUARD_SZ)
41
+ #define KERN_VM_SZ (SZ_4G + GUARD_SZ)
42
42
43
43
struct bpf_arena {
44
44
struct bpf_map map ;
@@ -110,7 +110,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
110
110
return ERR_PTR (- EINVAL );
111
111
112
112
vm_range = (u64 )attr -> max_entries * PAGE_SIZE ;
113
- if (vm_range > ( 1ull << 32 ) )
113
+ if (vm_range > SZ_4G )
114
114
return ERR_PTR (- E2BIG );
115
115
116
116
if ((attr -> map_extra >> 32 ) != ((attr -> map_extra + vm_range - 1 ) >> 32 ))
@@ -301,7 +301,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
301
301
302
302
if (pgoff )
303
303
return - EINVAL ;
304
- if (len > ( 1ull << 32 ) )
304
+ if (len > SZ_4G )
305
305
return - E2BIG ;
306
306
307
307
/* if user_vm_start was specified at arena creation time */
@@ -322,7 +322,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
322
322
if (WARN_ON_ONCE (arena -> user_vm_start ))
323
323
/* checks at map creation time should prevent this */
324
324
return - EFAULT ;
325
- return round_up (ret , 1ull << 32 );
325
+ return round_up (ret , SZ_4G );
326
326
}
327
327
328
328
static int arena_map_mmap (struct bpf_map * map , struct vm_area_struct * vma )
@@ -346,7 +346,7 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
346
346
return - EBUSY ;
347
347
348
348
/* Earlier checks should prevent this */
349
- if (WARN_ON_ONCE (vma -> vm_end - vma -> vm_start > ( 1ull << 32 ) || vma -> vm_pgoff ))
349
+ if (WARN_ON_ONCE (vma -> vm_end - vma -> vm_start > SZ_4G || vma -> vm_pgoff ))
350
350
return - EFAULT ;
351
351
352
352
if (remember_vma (arena , vma ))
@@ -420,7 +420,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
420
420
if (uaddr & ~PAGE_MASK )
421
421
return 0 ;
422
422
pgoff = compute_pgoff (arena , uaddr );
423
- if (pgoff + page_cnt > page_cnt_max )
423
+ if (pgoff > page_cnt_max - page_cnt )
424
424
/* requested address will be outside of user VMA */
425
425
return 0 ;
426
426
}
@@ -447,7 +447,13 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
447
447
goto out ;
448
448
449
449
uaddr32 = (u32 )(arena -> user_vm_start + pgoff * PAGE_SIZE );
450
- /* Earlier checks make sure that uaddr32 + page_cnt * PAGE_SIZE will not overflow 32-bit */
450
+ /* Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1
451
+ * will not overflow 32-bit. Lower 32-bit need to represent
452
+ * contiguous user address range.
453
+ * Map these pages at kern_vm_start base.
454
+ * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow
455
+ * lower 32-bit and it's ok.
456
+ */
451
457
ret = vm_area_map_pages (arena -> kern_vm , kern_vm_start + uaddr32 ,
452
458
kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE , pages );
453
459
if (ret ) {
@@ -510,6 +516,11 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
510
516
if (!page )
511
517
continue ;
512
518
if (page_cnt == 1 && page_mapped (page )) /* mapped by some user process */
519
+ /* Optimization for the common case of page_cnt==1:
520
+ * If page wasn't mapped into some user vma there
521
+ * is no need to call zap_pages which is slow. When
522
+ * page_cnt is big it's faster to do the batched zap.
523
+ */
513
524
zap_pages (arena , full_uaddr , 1 );
514
525
vm_area_unmap_pages (arena -> kern_vm , kaddr , kaddr + PAGE_SIZE );
515
526
__free_page (page );
0 commit comments