@@ -622,15 +622,19 @@ EXPORT_SYMBOL_GPL(reserve_iova);
622
622
/*
623
623
* As kmalloc's buffer size is fixed to power of 2, 127 is chosen to
624
624
* assure size of 'iova_magazine' to be 1024 bytes, so that no memory
625
- * will be wasted.
625
+ * will be wasted. Since only full magazines are inserted into the depot,
626
+ * we don't need to waste PFN capacity on a separate list head either.
626
627
*/
627
628
#define IOVA_MAG_SIZE 127
628
- #define MAX_GLOBAL_MAGS 32 /* magazines per bin */
629
629
630
630
struct iova_magazine {
631
- unsigned long size ;
631
+ union {
632
+ unsigned long size ;
633
+ struct iova_magazine * next ;
634
+ };
632
635
unsigned long pfns [IOVA_MAG_SIZE ];
633
636
};
637
+ static_assert (!(sizeof (struct iova_magazine ) & (sizeof (struct iova_magazine ) - 1 )));
634
638
635
639
struct iova_cpu_rcache {
636
640
spinlock_t lock ;
@@ -640,8 +644,7 @@ struct iova_cpu_rcache {
640
644
641
645
struct iova_rcache {
642
646
spinlock_t lock ;
643
- unsigned long depot_size ;
644
- struct iova_magazine * depot [MAX_GLOBAL_MAGS ];
647
+ struct iova_magazine * depot ;
645
648
struct iova_cpu_rcache __percpu * cpu_rcaches ;
646
649
};
647
650
@@ -717,6 +720,21 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
717
720
mag -> pfns [mag -> size ++ ] = pfn ;
718
721
}
719
722
723
+ static struct iova_magazine * iova_depot_pop (struct iova_rcache * rcache )
724
+ {
725
+ struct iova_magazine * mag = rcache -> depot ;
726
+
727
+ rcache -> depot = mag -> next ;
728
+ mag -> size = IOVA_MAG_SIZE ;
729
+ return mag ;
730
+ }
731
+
732
+ static void iova_depot_push (struct iova_rcache * rcache , struct iova_magazine * mag )
733
+ {
734
+ mag -> next = rcache -> depot ;
735
+ rcache -> depot = mag ;
736
+ }
737
+
720
738
int iova_domain_init_rcaches (struct iova_domain * iovad )
721
739
{
722
740
unsigned int cpu ;
@@ -734,7 +752,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad)
734
752
735
753
rcache = & iovad -> rcaches [i ];
736
754
spin_lock_init (& rcache -> lock );
737
- rcache -> depot_size = 0 ;
738
755
rcache -> cpu_rcaches = __alloc_percpu (sizeof (* cpu_rcache ),
739
756
cache_line_size ());
740
757
if (!rcache -> cpu_rcaches ) {
@@ -776,7 +793,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
776
793
struct iova_rcache * rcache ,
777
794
unsigned long iova_pfn )
778
795
{
779
- struct iova_magazine * mag_to_free = NULL ;
780
796
struct iova_cpu_rcache * cpu_rcache ;
781
797
bool can_insert = false;
782
798
unsigned long flags ;
@@ -794,12 +810,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
794
810
795
811
if (new_mag ) {
796
812
spin_lock (& rcache -> lock );
797
- if (rcache -> depot_size < MAX_GLOBAL_MAGS ) {
798
- rcache -> depot [rcache -> depot_size ++ ] =
799
- cpu_rcache -> loaded ;
800
- } else {
801
- mag_to_free = cpu_rcache -> loaded ;
802
- }
813
+ iova_depot_push (rcache , cpu_rcache -> loaded );
803
814
spin_unlock (& rcache -> lock );
804
815
805
816
cpu_rcache -> loaded = new_mag ;
@@ -812,11 +823,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
812
823
813
824
spin_unlock_irqrestore (& cpu_rcache -> lock , flags );
814
825
815
- if (mag_to_free ) {
816
- iova_magazine_free_pfns (mag_to_free , iovad );
817
- iova_magazine_free (mag_to_free );
818
- }
819
-
820
826
return can_insert ;
821
827
}
822
828
@@ -854,9 +860,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
854
860
has_pfn = true;
855
861
} else {
856
862
spin_lock (& rcache -> lock );
857
- if (rcache -> depot_size > 0 ) {
863
+ if (rcache -> depot ) {
858
864
iova_magazine_free (cpu_rcache -> loaded );
859
- cpu_rcache -> loaded = rcache -> depot [ -- rcache -> depot_size ] ;
865
+ cpu_rcache -> loaded = iova_depot_pop ( rcache ) ;
860
866
has_pfn = true;
861
867
}
862
868
spin_unlock (& rcache -> lock );
@@ -895,9 +901,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
895
901
struct iova_rcache * rcache ;
896
902
struct iova_cpu_rcache * cpu_rcache ;
897
903
unsigned int cpu ;
898
- int i , j ;
899
904
900
- for (i = 0 ; i < IOVA_RANGE_CACHE_MAX_SIZE ; ++ i ) {
905
+ for (int i = 0 ; i < IOVA_RANGE_CACHE_MAX_SIZE ; ++ i ) {
901
906
rcache = & iovad -> rcaches [i ];
902
907
if (!rcache -> cpu_rcaches )
903
908
break ;
@@ -907,8 +912,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
907
912
iova_magazine_free (cpu_rcache -> prev );
908
913
}
909
914
free_percpu (rcache -> cpu_rcaches );
910
- for ( j = 0 ; j < rcache -> depot_size ; ++ j )
911
- iova_magazine_free (rcache -> depot [ j ] );
915
+ while ( rcache -> depot )
916
+ iova_magazine_free (iova_depot_pop ( rcache ) );
912
917
}
913
918
914
919
kfree (iovad -> rcaches );
@@ -942,16 +947,16 @@ static void free_global_cached_iovas(struct iova_domain *iovad)
942
947
{
943
948
struct iova_rcache * rcache ;
944
949
unsigned long flags ;
945
- int i , j ;
946
950
947
- for (i = 0 ; i < IOVA_RANGE_CACHE_MAX_SIZE ; ++ i ) {
951
+ for (int i = 0 ; i < IOVA_RANGE_CACHE_MAX_SIZE ; ++ i ) {
948
952
rcache = & iovad -> rcaches [i ];
949
953
spin_lock_irqsave (& rcache -> lock , flags );
950
- for (j = 0 ; j < rcache -> depot_size ; ++ j ) {
951
- iova_magazine_free_pfns (rcache -> depot [j ], iovad );
952
- iova_magazine_free (rcache -> depot [j ]);
954
+ while (rcache -> depot ) {
955
+ struct iova_magazine * mag = iova_depot_pop (rcache );
956
+
957
+ iova_magazine_free_pfns (mag , iovad );
958
+ iova_magazine_free (mag );
953
959
}
954
- rcache -> depot_size = 0 ;
955
960
spin_unlock_irqrestore (& rcache -> lock , flags );
956
961
}
957
962
}
0 commit comments