Skip to content

Commit 644d2d6

Browse files
paulusmackpaulusmack
committed
Merge remote-tracking branch 'remotes/powerpc/topic/ppc-kvm' into kvm-ppc-next
This merges in the commits in the topic/ppc-kvm branch of the powerpc tree to get the changes to arch/powerpc which subsequent patches will rely on. Signed-off-by: Paul Mackerras <[email protected]>
2 parents 3762d45 + e5afdf9 commit 644d2d6

File tree

9 files changed

+142
-21
lines changed

9 files changed

+142
-21
lines changed

‎arch/powerpc/include/asm/iommu.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,11 @@ struct iommu_table_ops {
6464
long index,
6565
unsigned long *hpa,
6666
enum dma_data_direction *direction);
67+
/* Real mode */
68+
int (*exchange_rm)(struct iommu_table *tbl,
69+
long index,
70+
unsigned long *hpa,
71+
enum dma_data_direction *direction);
6772
#endif
6873
void (*clear)(struct iommu_table *tbl,
6974
long index, long npages);
@@ -114,6 +119,7 @@ struct iommu_table {
114119
struct list_head it_group_list;/* List of iommu_table_group_link */
115120
unsigned long *it_userspace; /* userspace view of the table */
116121
struct iommu_table_ops *it_ops;
122+
struct kref it_kref;
117123
};
118124

119125
#define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \
@@ -146,8 +152,8 @@ static inline void *get_iommu_table_base(struct device *dev)
146152

147153
extern int dma_iommu_dma_supported(struct device *dev, u64 mask);
148154

149-
/* Frees table for an individual device node */
150-
extern void iommu_free_table(struct iommu_table *tbl, const char *node_name);
155+
extern struct iommu_table *iommu_tce_table_get(struct iommu_table *tbl);
156+
extern int iommu_tce_table_put(struct iommu_table *tbl);
151157

152158
/* Initializes an iommu_table based in values set in the passed-in
153159
* structure
@@ -208,6 +214,8 @@ extern void iommu_del_device(struct device *dev);
208214
extern int __init tce_iommu_bus_notifier_init(void);
209215
extern long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry,
210216
unsigned long *hpa, enum dma_data_direction *direction);
217+
extern long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry,
218+
unsigned long *hpa, enum dma_data_direction *direction);
211219
#else
212220
static inline void iommu_register_group(struct iommu_table_group *table_group,
213221
int pci_domain_number,

‎arch/powerpc/include/asm/mmu_context.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,14 @@ extern void mm_iommu_init(struct mm_struct *mm);
2929
extern void mm_iommu_cleanup(struct mm_struct *mm);
3030
extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
3131
unsigned long ua, unsigned long size);
32+
extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(
33+
struct mm_struct *mm, unsigned long ua, unsigned long size);
3234
extern struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
3335
unsigned long ua, unsigned long entries);
3436
extern long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
3537
unsigned long ua, unsigned long *hpa);
38+
extern long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
39+
unsigned long ua, unsigned long *hpa);
3640
extern long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem);
3741
extern void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem);
3842
#endif

‎arch/powerpc/kernel/iommu.c

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -711,13 +711,16 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid)
711711
return tbl;
712712
}
713713

714-
void iommu_free_table(struct iommu_table *tbl, const char *node_name)
714+
static void iommu_table_free(struct kref *kref)
715715
{
716716
unsigned long bitmap_sz;
717717
unsigned int order;
718+
struct iommu_table *tbl;
718719

719-
if (!tbl)
720-
return;
720+
tbl = container_of(kref, struct iommu_table, it_kref);
721+
722+
if (tbl->it_ops->free)
723+
tbl->it_ops->free(tbl);
721724

722725
if (!tbl->it_map) {
723726
kfree(tbl);
@@ -733,7 +736,7 @@ void iommu_free_table(struct iommu_table *tbl, const char *node_name)
733736

734737
/* verify that table contains no entries */
735738
if (!bitmap_empty(tbl->it_map, tbl->it_size))
736-
pr_warn("%s: Unexpected TCEs for %s\n", __func__, node_name);
739+
pr_warn("%s: Unexpected TCEs\n", __func__);
737740

738741
/* calculate bitmap size in bytes */
739742
bitmap_sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
@@ -746,6 +749,24 @@ void iommu_free_table(struct iommu_table *tbl, const char *node_name)
746749
kfree(tbl);
747750
}
748751

752+
struct iommu_table *iommu_tce_table_get(struct iommu_table *tbl)
753+
{
754+
if (kref_get_unless_zero(&tbl->it_kref))
755+
return tbl;
756+
757+
return NULL;
758+
}
759+
EXPORT_SYMBOL_GPL(iommu_tce_table_get);
760+
761+
int iommu_tce_table_put(struct iommu_table *tbl)
762+
{
763+
if (WARN_ON(!tbl))
764+
return 0;
765+
766+
return kref_put(&tbl->it_kref, iommu_table_free);
767+
}
768+
EXPORT_SYMBOL_GPL(iommu_tce_table_put);
769+
749770
/* Creates TCEs for a user provided buffer. The user buffer must be
750771
* contiguous real kernel storage (not vmalloc). The address passed here
751772
* comprises a page address and offset into that page. The dma_addr_t
@@ -1004,6 +1025,31 @@ long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry,
10041025
}
10051026
EXPORT_SYMBOL_GPL(iommu_tce_xchg);
10061027

1028+
#ifdef CONFIG_PPC_BOOK3S_64
1029+
long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry,
1030+
unsigned long *hpa, enum dma_data_direction *direction)
1031+
{
1032+
long ret;
1033+
1034+
ret = tbl->it_ops->exchange_rm(tbl, entry, hpa, direction);
1035+
1036+
if (!ret && ((*direction == DMA_FROM_DEVICE) ||
1037+
(*direction == DMA_BIDIRECTIONAL))) {
1038+
struct page *pg = realmode_pfn_to_page(*hpa >> PAGE_SHIFT);
1039+
1040+
if (likely(pg)) {
1041+
SetPageDirty(pg);
1042+
} else {
1043+
tbl->it_ops->exchange_rm(tbl, entry, hpa, direction);
1044+
ret = -EFAULT;
1045+
}
1046+
}
1047+
1048+
return ret;
1049+
}
1050+
EXPORT_SYMBOL_GPL(iommu_tce_xchg_rm);
1051+
#endif
1052+
10071053
int iommu_take_ownership(struct iommu_table *tbl)
10081054
{
10091055
unsigned long flags, i, sz = (tbl->it_size + 7) >> 3;

‎arch/powerpc/mm/mmu_context_iommu.c

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,25 @@ struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
314314
}
315315
EXPORT_SYMBOL_GPL(mm_iommu_lookup);
316316

317+
struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(struct mm_struct *mm,
318+
unsigned long ua, unsigned long size)
319+
{
320+
struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
321+
322+
list_for_each_entry_lockless(mem, &mm->context.iommu_group_mem_list,
323+
next) {
324+
if ((mem->ua <= ua) &&
325+
(ua + size <= mem->ua +
326+
(mem->entries << PAGE_SHIFT))) {
327+
ret = mem;
328+
break;
329+
}
330+
}
331+
332+
return ret;
333+
}
334+
EXPORT_SYMBOL_GPL(mm_iommu_lookup_rm);
335+
317336
struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
318337
unsigned long ua, unsigned long entries)
319338
{
@@ -345,6 +364,26 @@ long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
345364
}
346365
EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa);
347366

367+
long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
368+
unsigned long ua, unsigned long *hpa)
369+
{
370+
const long entry = (ua - mem->ua) >> PAGE_SHIFT;
371+
void *va = &mem->hpas[entry];
372+
unsigned long *pa;
373+
374+
if (entry >= mem->entries)
375+
return -EFAULT;
376+
377+
pa = (void *) vmalloc_to_phys(va);
378+
if (!pa)
379+
return -EFAULT;
380+
381+
*hpa = *pa | (ua & ~PAGE_MASK);
382+
383+
return 0;
384+
}
385+
EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa_rm);
386+
348387
long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem)
349388
{
350389
if (atomic64_inc_not_zero(&mem->mapped))

‎arch/powerpc/platforms/powernv/pci-ioda.c

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1424,8 +1424,7 @@ static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe
14241424
iommu_group_put(pe->table_group.group);
14251425
BUG_ON(pe->table_group.group);
14261426
}
1427-
pnv_pci_ioda2_table_free_pages(tbl);
1428-
iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
1427+
iommu_tce_table_put(tbl);
14291428
}
14301429

14311430
static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
@@ -1860,6 +1859,17 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
18601859

18611860
return ret;
18621861
}
1862+
1863+
static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index,
1864+
unsigned long *hpa, enum dma_data_direction *direction)
1865+
{
1866+
long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1867+
1868+
if (!ret)
1869+
pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true);
1870+
1871+
return ret;
1872+
}
18631873
#endif
18641874

18651875
static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
@@ -1874,6 +1884,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = {
18741884
.set = pnv_ioda1_tce_build,
18751885
#ifdef CONFIG_IOMMU_API
18761886
.exchange = pnv_ioda1_tce_xchg,
1887+
.exchange_rm = pnv_ioda1_tce_xchg_rm,
18771888
#endif
18781889
.clear = pnv_ioda1_tce_free,
18791890
.get = pnv_tce_get,
@@ -1948,7 +1959,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
19481959
{
19491960
struct iommu_table_group_link *tgl;
19501961

1951-
list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1962+
list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {
19521963
struct pnv_ioda_pe *pe = container_of(tgl->table_group,
19531964
struct pnv_ioda_pe, table_group);
19541965
struct pnv_phb *phb = pe->phb;
@@ -2004,6 +2015,17 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
20042015

20052016
return ret;
20062017
}
2018+
2019+
static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index,
2020+
unsigned long *hpa, enum dma_data_direction *direction)
2021+
{
2022+
long ret = pnv_tce_xchg(tbl, index, hpa, direction);
2023+
2024+
if (!ret)
2025+
pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true);
2026+
2027+
return ret;
2028+
}
20072029
#endif
20082030

20092031
static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
@@ -2017,13 +2039,13 @@ static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
20172039
static void pnv_ioda2_table_free(struct iommu_table *tbl)
20182040
{
20192041
pnv_pci_ioda2_table_free_pages(tbl);
2020-
iommu_free_table(tbl, "pnv");
20212042
}
20222043

20232044
static struct iommu_table_ops pnv_ioda2_iommu_ops = {
20242045
.set = pnv_ioda2_tce_build,
20252046
#ifdef CONFIG_IOMMU_API
20262047
.exchange = pnv_ioda2_tce_xchg,
2048+
.exchange_rm = pnv_ioda2_tce_xchg_rm,
20272049
#endif
20282050
.clear = pnv_ioda2_tce_free,
20292051
.get = pnv_tce_get,
@@ -2203,7 +2225,7 @@ static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb,
22032225
__free_pages(tce_mem, get_order(tce32_segsz * segs));
22042226
if (tbl) {
22052227
pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
2206-
iommu_free_table(tbl, "pnv");
2228+
iommu_tce_table_put(tbl);
22072229
}
22082230
}
22092231

@@ -2293,16 +2315,16 @@ static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
22932315
if (!tbl)
22942316
return -ENOMEM;
22952317

2318+
tbl->it_ops = &pnv_ioda2_iommu_ops;
2319+
22962320
ret = pnv_pci_ioda2_table_alloc_pages(nid,
22972321
bus_offset, page_shift, window_size,
22982322
levels, tbl);
22992323
if (ret) {
2300-
iommu_free_table(tbl, "pnv");
2324+
iommu_tce_table_put(tbl);
23012325
return ret;
23022326
}
23032327

2304-
tbl->it_ops = &pnv_ioda2_iommu_ops;
2305-
23062328
*ptbl = tbl;
23072329

23082330
return 0;
@@ -2343,7 +2365,7 @@ static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
23432365
if (rc) {
23442366
pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
23452367
rc);
2346-
pnv_ioda2_table_free(tbl);
2368+
iommu_tce_table_put(tbl);
23472369
return rc;
23482370
}
23492371

@@ -2431,7 +2453,7 @@ static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
24312453
pnv_pci_ioda2_unset_window(&pe->table_group, 0);
24322454
if (pe->pbus)
24332455
pnv_ioda_setup_bus_dma(pe, pe->pbus, false);
2434-
pnv_ioda2_table_free(tbl);
2456+
iommu_tce_table_put(tbl);
24352457
}
24362458

24372459
static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
@@ -3406,7 +3428,7 @@ static void pnv_pci_ioda1_release_pe_dma(struct pnv_ioda_pe *pe)
34063428
}
34073429

34083430
free_pages(tbl->it_base, get_order(tbl->it_size << 3));
3409-
iommu_free_table(tbl, "pnv");
3431+
iommu_tce_table_put(tbl);
34103432
}
34113433

34123434
static void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
@@ -3433,7 +3455,7 @@ static void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
34333455
}
34343456

34353457
pnv_pci_ioda2_table_free_pages(tbl);
3436-
iommu_free_table(tbl, "pnv");
3458+
iommu_tce_table_put(tbl);
34373459
}
34383460

34393461
static void pnv_ioda_free_pe_seg(struct pnv_ioda_pe *pe,

‎arch/powerpc/platforms/powernv/pci.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,7 @@ struct iommu_table *pnv_pci_table_alloc(int nid)
767767

768768
tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, nid);
769769
INIT_LIST_HEAD_RCU(&tbl->it_group_list);
770+
kref_init(&tbl->it_kref);
770771

771772
return tbl;
772773
}

‎arch/powerpc/platforms/pseries/iommu.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ static struct iommu_table_group *iommu_pseries_alloc_group(int node)
7474
goto fail_exit;
7575

7676
INIT_LIST_HEAD_RCU(&tbl->it_group_list);
77+
kref_init(&tbl->it_kref);
7778
tgl->table_group = table_group;
7879
list_add_rcu(&tgl->next, &tbl->it_group_list);
7980

@@ -115,7 +116,7 @@ static void iommu_pseries_free_group(struct iommu_table_group *table_group,
115116
BUG_ON(table_group->group);
116117
}
117118
#endif
118-
iommu_free_table(tbl, node_name);
119+
iommu_tce_table_put(tbl);
119120

120121
kfree(table_group);
121122
}

‎arch/powerpc/platforms/pseries/vio.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1318,7 +1318,7 @@ static void vio_dev_release(struct device *dev)
13181318
struct iommu_table *tbl = get_iommu_table_base(dev);
13191319

13201320
if (tbl)
1321-
iommu_free_table(tbl, of_node_full_name(dev->of_node));
1321+
iommu_tce_table_put(tbl);
13221322
of_node_put(dev->of_node);
13231323
kfree(to_vio_dev(dev));
13241324
}

‎drivers/vfio/vfio_iommu_spapr_tce.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -680,7 +680,7 @@ static void tce_iommu_free_table(struct tce_container *container,
680680
unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
681681

682682
tce_iommu_userspace_view_free(tbl, container->mm);
683-
tbl->it_ops->free(tbl);
683+
iommu_tce_table_put(tbl);
684684
decrement_locked_vm(container->mm, pages);
685685
}
686686

0 commit comments

Comments
 (0)