Skip to content

Commit fa4c450

Browse files
niklas88joergroedel
niklas88
authored and
joergroedel
committed
iommu: Allow .iotlb_sync_map to fail and handle s390's -ENOMEM return
On s390 when using a paging hypervisor, .iotlb_sync_map is used to sync mappings by letting the hypervisor inspect the synced IOVA range and updating a shadow table. This however means that .iotlb_sync_map can fail as the hypervisor may run out of resources while doing the sync. This can be due to the hypervisor being unable to pin guest pages, due to a limit on mapped addresses such as vfio_iommu_type1.dma_entry_limit or lack of other resources. Either way such a failure to sync a mapping should result in a DMA_MAPPING_ERROR. Now especially when running with batched IOTLB flushes for unmap it may be that some IOVAs have already been invalidated but not yet synced via .iotlb_sync_map. Thus if the hypervisor indicates running out of resources, first do a global flush allowing the hypervisor to free resources associated with these mappings as well a retry creating the new mappings and only if that also fails report this error to callers. Reviewed-by: Lu Baolu <[email protected]> Reviewed-by: Matthew Rosato <[email protected]> Acked-by: Jernej Skrabec <[email protected]> # sun50i Signed-off-by: Niklas Schnelle <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Joerg Roedel <[email protected]>
1 parent ccb76c5 commit fa4c450

File tree

10 files changed

+63
-26
lines changed

10 files changed

+63
-26
lines changed

‎drivers/iommu/amd/iommu.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2233,14 +2233,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
22332233
return ret;
22342234
}
22352235

2236-
static void amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
2237-
unsigned long iova, size_t size)
2236+
static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
2237+
unsigned long iova, size_t size)
22382238
{
22392239
struct protection_domain *domain = to_pdomain(dom);
22402240
struct io_pgtable_ops *ops = &domain->iop.iop.ops;
22412241

22422242
if (ops->map_pages)
22432243
domain_flush_np_cache(domain, iova, size);
2244+
return 0;
22442245
}
22452246

22462247
static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova,

‎drivers/iommu/apple-dart.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -506,10 +506,11 @@ static void apple_dart_iotlb_sync(struct iommu_domain *domain,
506506
apple_dart_domain_flush_tlb(to_dart_domain(domain));
507507
}
508508

509-
static void apple_dart_iotlb_sync_map(struct iommu_domain *domain,
510-
unsigned long iova, size_t size)
509+
static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
510+
unsigned long iova, size_t size)
511511
{
512512
apple_dart_domain_flush_tlb(to_dart_domain(domain));
513+
return 0;
513514
}
514515

515516
static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,

‎drivers/iommu/intel/iommu.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4678,8 +4678,8 @@ static bool risky_device(struct pci_dev *pdev)
46784678
return false;
46794679
}
46804680

4681-
static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4682-
unsigned long iova, size_t size)
4681+
static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4682+
unsigned long iova, size_t size)
46834683
{
46844684
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
46854685
unsigned long pages = aligned_nrpages(iova, size);
@@ -4689,6 +4689,7 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
46894689

46904690
xa_for_each(&dmar_domain->iommu_array, i, info)
46914691
__mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
4692+
return 0;
46924693
}
46934694

46944695
static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)

‎drivers/iommu/iommu.c

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2585,8 +2585,17 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
25852585
return -EINVAL;
25862586

25872587
ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
2588-
if (ret == 0 && ops->iotlb_sync_map)
2589-
ops->iotlb_sync_map(domain, iova, size);
2588+
if (ret == 0 && ops->iotlb_sync_map) {
2589+
ret = ops->iotlb_sync_map(domain, iova, size);
2590+
if (ret)
2591+
goto out_err;
2592+
}
2593+
2594+
return ret;
2595+
2596+
out_err:
2597+
/* undo mappings already done */
2598+
iommu_unmap(domain, iova, size);
25902599

25912600
return ret;
25922601
}
@@ -2714,8 +2723,11 @@ ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
27142723
sg = sg_next(sg);
27152724
}
27162725

2717-
if (ops->iotlb_sync_map)
2718-
ops->iotlb_sync_map(domain, iova, mapped);
2726+
if (ops->iotlb_sync_map) {
2727+
ret = ops->iotlb_sync_map(domain, iova, mapped);
2728+
if (ret)
2729+
goto out_err;
2730+
}
27192731
return mapped;
27202732

27212733
out_err:

‎drivers/iommu/msm_iommu.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -498,12 +498,13 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
498498
return ret;
499499
}
500500

501-
static void msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
502-
size_t size)
501+
static int msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
502+
size_t size)
503503
{
504504
struct msm_priv *priv = to_msm_priv(domain);
505505

506506
__flush_iotlb_range(iova, size, SZ_4K, false, priv);
507+
return 0;
507508
}
508509

509510
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,

‎drivers/iommu/mtk_iommu.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -837,12 +837,13 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
837837
mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->bank);
838838
}
839839

840-
static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
841-
size_t size)
840+
static int mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
841+
size_t size)
842842
{
843843
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
844844

845845
mtk_iommu_tlb_flush_range_sync(iova, size, dom->bank);
846+
return 0;
846847
}
847848

848849
static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,

‎drivers/iommu/s390-iommu.c

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -219,15 +219,20 @@ static void s390_iommu_release_device(struct device *dev)
219219
__s390_iommu_detach_device(zdev);
220220
}
221221

222+
static int zpci_refresh_all(struct zpci_dev *zdev)
223+
{
224+
return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
225+
zdev->end_dma - zdev->start_dma + 1);
226+
}
227+
222228
static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain)
223229
{
224230
struct s390_domain *s390_domain = to_s390_domain(domain);
225231
struct zpci_dev *zdev;
226232

227233
rcu_read_lock();
228234
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
229-
zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
230-
zdev->end_dma - zdev->start_dma + 1);
235+
zpci_refresh_all(zdev);
231236
}
232237
rcu_read_unlock();
233238
}
@@ -251,20 +256,32 @@ static void s390_iommu_iotlb_sync(struct iommu_domain *domain,
251256
rcu_read_unlock();
252257
}
253258

254-
static void s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
255-
unsigned long iova, size_t size)
259+
static int s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
260+
unsigned long iova, size_t size)
256261
{
257262
struct s390_domain *s390_domain = to_s390_domain(domain);
258263
struct zpci_dev *zdev;
264+
int ret = 0;
259265

260266
rcu_read_lock();
261267
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
262268
if (!zdev->tlb_refresh)
263269
continue;
264-
zpci_refresh_trans((u64)zdev->fh << 32,
265-
iova, size);
270+
ret = zpci_refresh_trans((u64)zdev->fh << 32,
271+
iova, size);
272+
/*
273+
* let the hypervisor discover invalidated entries
274+
* allowing it to free IOVAs and unpin pages
275+
*/
276+
if (ret == -ENOMEM) {
277+
ret = zpci_refresh_all(zdev);
278+
if (ret)
279+
break;
280+
}
266281
}
267282
rcu_read_unlock();
283+
284+
return ret;
268285
}
269286

270287
static int s390_iommu_validate_trans(struct s390_domain *s390_domain,

‎drivers/iommu/sprd-iommu.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,8 +341,8 @@ static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
341341
return size;
342342
}
343343

344-
static void sprd_iommu_sync_map(struct iommu_domain *domain,
345-
unsigned long iova, size_t size)
344+
static int sprd_iommu_sync_map(struct iommu_domain *domain,
345+
unsigned long iova, size_t size)
346346
{
347347
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
348348
unsigned int reg;
@@ -354,6 +354,7 @@ static void sprd_iommu_sync_map(struct iommu_domain *domain,
354354

355355
/* clear IOMMU TLB buffer after page table updated */
356356
sprd_iommu_write(dom->sdev, reg, 0xffffffff);
357+
return 0;
357358
}
358359

359360
static void sprd_iommu_sync(struct iommu_domain *domain,

‎drivers/iommu/sun50i-iommu.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -401,8 +401,8 @@ static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
401401
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
402402
}
403403

404-
static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
405-
unsigned long iova, size_t size)
404+
static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
405+
unsigned long iova, size_t size)
406406
{
407407
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
408408
struct sun50i_iommu *iommu = sun50i_domain->iommu;
@@ -411,6 +411,8 @@ static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
411411
spin_lock_irqsave(&iommu->iommu_lock, flags);
412412
sun50i_iommu_zap_range(iommu, iova, size);
413413
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
414+
415+
return 0;
414416
}
415417

416418
static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,

‎include/linux/iommu.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -350,8 +350,8 @@ struct iommu_domain_ops {
350350
struct iommu_iotlb_gather *iotlb_gather);
351351

352352
void (*flush_iotlb_all)(struct iommu_domain *domain);
353-
void (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
354-
size_t size);
353+
int (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
354+
size_t size);
355355
void (*iotlb_sync)(struct iommu_domain *domain,
356356
struct iommu_iotlb_gather *iotlb_gather);
357357

0 commit comments

Comments
 (0)