Skip to content

Commit 911aa12

Browse files
rmurphy-armjoergroedel
authored andcommitted
iommu/iova: Make the rcache depot scale better
The algorithm in the original paper specifies the storage of full magazines in the depot as an unbounded list rather than a fixed-size array. It turns out to be pretty straightforward to do this in our implementation with no significant loss of efficiency. This allows the depot to scale up to the working set sizes of larger systems, while also potentially saving some memory on smaller ones too. Since this involves touching struct iova_magazine with the requisite care, we may as well reinforce the comment with a proper assertion too. Reviewed-by: John Garry <[email protected]> Reviewed-by: Jerry Snitselaar <[email protected]> Signed-off-by: Robin Murphy <[email protected]> Link: https://lore.kernel.org/r/f597aa72fc3e1d315bc4574af0ce0ebe5c31cd22.1694535580.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel <[email protected]>
1 parent afad94a commit 911aa12

File tree

1 file changed

+35
-30
lines changed

1 file changed

+35
-30
lines changed

drivers/iommu/iova.c

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -622,15 +622,19 @@ EXPORT_SYMBOL_GPL(reserve_iova);
622622
/*
623623
* As kmalloc's buffer size is fixed to power of 2, 127 is chosen to
624624
* assure size of 'iova_magazine' to be 1024 bytes, so that no memory
625-
* will be wasted.
625+
* will be wasted. Since only full magazines are inserted into the depot,
626+
* we don't need to waste PFN capacity on a separate list head either.
626627
*/
627628
#define IOVA_MAG_SIZE 127
628-
#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
629629

630630
struct iova_magazine {
631-
unsigned long size;
631+
union {
632+
unsigned long size;
633+
struct iova_magazine *next;
634+
};
632635
unsigned long pfns[IOVA_MAG_SIZE];
633636
};
637+
static_assert(!(sizeof(struct iova_magazine) & (sizeof(struct iova_magazine) - 1)));
634638

635639
struct iova_cpu_rcache {
636640
spinlock_t lock;
@@ -640,8 +644,7 @@ struct iova_cpu_rcache {
640644

641645
struct iova_rcache {
642646
spinlock_t lock;
643-
unsigned long depot_size;
644-
struct iova_magazine *depot[MAX_GLOBAL_MAGS];
647+
struct iova_magazine *depot;
645648
struct iova_cpu_rcache __percpu *cpu_rcaches;
646649
};
647650

@@ -717,6 +720,21 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
717720
mag->pfns[mag->size++] = pfn;
718721
}
719722

723+
static struct iova_magazine *iova_depot_pop(struct iova_rcache *rcache)
724+
{
725+
struct iova_magazine *mag = rcache->depot;
726+
727+
rcache->depot = mag->next;
728+
mag->size = IOVA_MAG_SIZE;
729+
return mag;
730+
}
731+
732+
static void iova_depot_push(struct iova_rcache *rcache, struct iova_magazine *mag)
733+
{
734+
mag->next = rcache->depot;
735+
rcache->depot = mag;
736+
}
737+
720738
int iova_domain_init_rcaches(struct iova_domain *iovad)
721739
{
722740
unsigned int cpu;
@@ -734,7 +752,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad)
734752

735753
rcache = &iovad->rcaches[i];
736754
spin_lock_init(&rcache->lock);
737-
rcache->depot_size = 0;
738755
rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
739756
cache_line_size());
740757
if (!rcache->cpu_rcaches) {
@@ -776,7 +793,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
776793
struct iova_rcache *rcache,
777794
unsigned long iova_pfn)
778795
{
779-
struct iova_magazine *mag_to_free = NULL;
780796
struct iova_cpu_rcache *cpu_rcache;
781797
bool can_insert = false;
782798
unsigned long flags;
@@ -794,12 +810,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
794810

795811
if (new_mag) {
796812
spin_lock(&rcache->lock);
797-
if (rcache->depot_size < MAX_GLOBAL_MAGS) {
798-
rcache->depot[rcache->depot_size++] =
799-
cpu_rcache->loaded;
800-
} else {
801-
mag_to_free = cpu_rcache->loaded;
802-
}
813+
iova_depot_push(rcache, cpu_rcache->loaded);
803814
spin_unlock(&rcache->lock);
804815

805816
cpu_rcache->loaded = new_mag;
@@ -812,11 +823,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
812823

813824
spin_unlock_irqrestore(&cpu_rcache->lock, flags);
814825

815-
if (mag_to_free) {
816-
iova_magazine_free_pfns(mag_to_free, iovad);
817-
iova_magazine_free(mag_to_free);
818-
}
819-
820826
return can_insert;
821827
}
822828

@@ -854,9 +860,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
854860
has_pfn = true;
855861
} else {
856862
spin_lock(&rcache->lock);
857-
if (rcache->depot_size > 0) {
863+
if (rcache->depot) {
858864
iova_magazine_free(cpu_rcache->loaded);
859-
cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
865+
cpu_rcache->loaded = iova_depot_pop(rcache);
860866
has_pfn = true;
861867
}
862868
spin_unlock(&rcache->lock);
@@ -895,9 +901,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
895901
struct iova_rcache *rcache;
896902
struct iova_cpu_rcache *cpu_rcache;
897903
unsigned int cpu;
898-
int i, j;
899904

900-
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
905+
for (int i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
901906
rcache = &iovad->rcaches[i];
902907
if (!rcache->cpu_rcaches)
903908
break;
@@ -907,8 +912,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
907912
iova_magazine_free(cpu_rcache->prev);
908913
}
909914
free_percpu(rcache->cpu_rcaches);
910-
for (j = 0; j < rcache->depot_size; ++j)
911-
iova_magazine_free(rcache->depot[j]);
915+
while (rcache->depot)
916+
iova_magazine_free(iova_depot_pop(rcache));
912917
}
913918

914919
kfree(iovad->rcaches);
@@ -942,16 +947,16 @@ static void free_global_cached_iovas(struct iova_domain *iovad)
942947
{
943948
struct iova_rcache *rcache;
944949
unsigned long flags;
945-
int i, j;
946950

947-
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
951+
for (int i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
948952
rcache = &iovad->rcaches[i];
949953
spin_lock_irqsave(&rcache->lock, flags);
950-
for (j = 0; j < rcache->depot_size; ++j) {
951-
iova_magazine_free_pfns(rcache->depot[j], iovad);
952-
iova_magazine_free(rcache->depot[j]);
954+
while (rcache->depot) {
955+
struct iova_magazine *mag = iova_depot_pop(rcache);
956+
957+
iova_magazine_free_pfns(mag, iovad);
958+
iova_magazine_free(mag);
953959
}
954-
rcache->depot_size = 0;
955960
spin_unlock_irqrestore(&rcache->lock, flags);
956961
}
957962
}

0 commit comments

Comments
 (0)