Skip to content

Commit 6aa63a4

Browse files
rmurphy-armjgunthorpe
authored andcommitted
iommu: Sort out domain user data
When DMA/MSI cookies were made first-class citizens back in commit 46983fc ("iommu: Pull IOVA cookie management into the core"), there was no real need to further expose the two different cookie types. However, now that IOMMUFD wants to add a third type of MSI-mapping cookie, we do have a nicely compelling reason to properly dismabiguate things at the domain level beyond just vaguely guessing from the domain type. Meanwhile, we also effectively have another "cookie" in the form of the anonymous union for other user data, which isn't much better in terms of being vague and unenforced. The fact is that all these cookie types are mutually exclusive, in the sense that combining them makes zero sense and/or would be catastrophic (iommu_set_fault_handler() on an SVA domain, anyone?) - the only combination which *might* be reasonable is perhaps a fault handler and an MSI cookie, but nobody's doing that at the moment, so let's rule it out as well for the sake of being clear and robust. To that end, we pull DMA and MSI cookies apart a little more, mostly to clear up the ambiguity at domain teardown, then for clarity (and to save a little space), move them into the union, whose ownership we can then properly describe and enforce entirely unambiguously. [nicolinc: rebase on latest tree; use prefix IOMMU_COOKIE_; merge unions in iommu_domain; add IOMMU_COOKIE_IOMMUFD for iommufd_hwpt] Link: https://patch.msgid.link/r/1ace9076c95204bbe193ee77499d395f15f44b23.1742871535.git.nicolinc@nvidia.com Signed-off-by: Robin Murphy <[email protected]> Reviewed-by: Kevin Tian <[email protected]> Signed-off-by: Nicolin Chen <[email protected]> Reviewed-by: Jason Gunthorpe <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
1 parent da0c565 commit 6aa63a4

File tree

6 files changed

+143
-98
lines changed

6 files changed

+143
-98
lines changed

drivers/iommu/dma-iommu.c

Lines changed: 104 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,6 @@ struct iommu_dma_msi_page {
4242
phys_addr_t phys;
4343
};
4444

45-
enum iommu_dma_cookie_type {
46-
IOMMU_DMA_IOVA_COOKIE,
47-
IOMMU_DMA_MSI_COOKIE,
48-
};
49-
5045
enum iommu_dma_queue_type {
5146
IOMMU_DMA_OPTS_PER_CPU_QUEUE,
5247
IOMMU_DMA_OPTS_SINGLE_QUEUE,
@@ -59,35 +54,31 @@ struct iommu_dma_options {
5954
};
6055

6156
struct iommu_dma_cookie {
62-
enum iommu_dma_cookie_type type;
57+
struct iova_domain iovad;
58+
struct list_head msi_page_list;
59+
/* Flush queue */
6360
union {
64-
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
65-
struct {
66-
struct iova_domain iovad;
67-
/* Flush queue */
68-
union {
69-
struct iova_fq *single_fq;
70-
struct iova_fq __percpu *percpu_fq;
71-
};
72-
/* Number of TLB flushes that have been started */
73-
atomic64_t fq_flush_start_cnt;
74-
/* Number of TLB flushes that have been finished */
75-
atomic64_t fq_flush_finish_cnt;
76-
/* Timer to regularily empty the flush queues */
77-
struct timer_list fq_timer;
78-
/* 1 when timer is active, 0 when not */
79-
atomic_t fq_timer_on;
80-
};
81-
/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
82-
dma_addr_t msi_iova;
61+
struct iova_fq *single_fq;
62+
struct iova_fq __percpu *percpu_fq;
8363
};
84-
struct list_head msi_page_list;
85-
64+
/* Number of TLB flushes that have been started */
65+
atomic64_t fq_flush_start_cnt;
66+
/* Number of TLB flushes that have been finished */
67+
atomic64_t fq_flush_finish_cnt;
68+
/* Timer to regularily empty the flush queues */
69+
struct timer_list fq_timer;
70+
/* 1 when timer is active, 0 when not */
71+
atomic_t fq_timer_on;
8672
/* Domain for flush queue callback; NULL if flush queue not in use */
87-
struct iommu_domain *fq_domain;
73+
struct iommu_domain *fq_domain;
8874
/* Options for dma-iommu use */
89-
struct iommu_dma_options options;
90-
struct mutex mutex;
75+
struct iommu_dma_options options;
76+
struct mutex mutex;
77+
};
78+
79+
struct iommu_dma_msi_cookie {
80+
dma_addr_t msi_iova;
81+
struct list_head msi_page_list;
9182
};
9283

9384
static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
@@ -369,40 +360,26 @@ int iommu_dma_init_fq(struct iommu_domain *domain)
369360
return 0;
370361
}
371362

372-
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
373-
{
374-
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
375-
return cookie->iovad.granule;
376-
return PAGE_SIZE;
377-
}
378-
379-
static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
380-
{
381-
struct iommu_dma_cookie *cookie;
382-
383-
cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
384-
if (cookie) {
385-
INIT_LIST_HEAD(&cookie->msi_page_list);
386-
cookie->type = type;
387-
}
388-
return cookie;
389-
}
390-
391363
/**
392364
* iommu_get_dma_cookie - Acquire DMA-API resources for a domain
393365
* @domain: IOMMU domain to prepare for DMA-API usage
394366
*/
395367
int iommu_get_dma_cookie(struct iommu_domain *domain)
396368
{
397-
if (domain->iova_cookie)
369+
struct iommu_dma_cookie *cookie;
370+
371+
if (domain->cookie_type != IOMMU_COOKIE_NONE)
398372
return -EEXIST;
399373

400-
domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);
401-
if (!domain->iova_cookie)
374+
cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
375+
if (!cookie)
402376
return -ENOMEM;
403377

404-
mutex_init(&domain->iova_cookie->mutex);
378+
mutex_init(&cookie->mutex);
379+
INIT_LIST_HEAD(&cookie->msi_page_list);
405380
iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);
381+
domain->cookie_type = IOMMU_COOKIE_DMA_IOVA;
382+
domain->iova_cookie = cookie;
406383
return 0;
407384
}
408385

@@ -420,29 +397,30 @@ int iommu_get_dma_cookie(struct iommu_domain *domain)
420397
*/
421398
int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
422399
{
423-
struct iommu_dma_cookie *cookie;
400+
struct iommu_dma_msi_cookie *cookie;
424401

425402
if (domain->type != IOMMU_DOMAIN_UNMANAGED)
426403
return -EINVAL;
427404

428-
if (domain->iova_cookie)
405+
if (domain->cookie_type != IOMMU_COOKIE_NONE)
429406
return -EEXIST;
430407

431-
cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
408+
cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
432409
if (!cookie)
433410
return -ENOMEM;
434411

435412
cookie->msi_iova = base;
436-
domain->iova_cookie = cookie;
413+
INIT_LIST_HEAD(&cookie->msi_page_list);
437414
iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);
415+
domain->cookie_type = IOMMU_COOKIE_DMA_MSI;
416+
domain->msi_cookie = cookie;
438417
return 0;
439418
}
440419
EXPORT_SYMBOL(iommu_get_msi_cookie);
441420

442421
/**
443422
* iommu_put_dma_cookie - Release a domain's DMA mapping resources
444-
* @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or
445-
* iommu_get_msi_cookie()
423+
* @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
446424
*/
447425
void iommu_put_dma_cookie(struct iommu_domain *domain)
448426
{
@@ -454,20 +432,27 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
454432
return;
455433
#endif
456434

457-
if (!cookie)
458-
return;
459-
460-
if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
435+
if (cookie->iovad.granule) {
461436
iommu_dma_free_fq(cookie);
462437
put_iova_domain(&cookie->iovad);
463438
}
439+
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list)
440+
kfree(msi);
441+
kfree(cookie);
442+
}
464443

465-
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
466-
list_del(&msi->list);
444+
/**
445+
* iommu_put_msi_cookie - Release a domain's MSI mapping resources
446+
* @domain: IOMMU domain previously prepared by iommu_get_msi_cookie()
447+
*/
448+
void iommu_put_msi_cookie(struct iommu_domain *domain)
449+
{
450+
struct iommu_dma_msi_cookie *cookie = domain->msi_cookie;
451+
struct iommu_dma_msi_page *msi, *tmp;
452+
453+
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list)
467454
kfree(msi);
468-
}
469455
kfree(cookie);
470-
domain->iova_cookie = NULL;
471456
}
472457

473458
/**
@@ -687,7 +672,7 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev
687672
struct iova_domain *iovad;
688673
int ret;
689674

690-
if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
675+
if (!cookie || domain->cookie_type != IOMMU_COOKIE_DMA_IOVA)
691676
return -EINVAL;
692677

693678
iovad = &cookie->iovad;
@@ -777,9 +762,9 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
777762
struct iova_domain *iovad = &cookie->iovad;
778763
unsigned long shift, iova_len, iova;
779764

780-
if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
781-
cookie->msi_iova += size;
782-
return cookie->msi_iova - size;
765+
if (domain->cookie_type == IOMMU_COOKIE_DMA_MSI) {
766+
domain->msi_cookie->msi_iova += size;
767+
return domain->msi_cookie->msi_iova - size;
783768
}
784769

785770
shift = iova_shift(iovad);
@@ -816,16 +801,16 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
816801
return (dma_addr_t)iova << shift;
817802
}
818803

819-
static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
820-
dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
804+
static void iommu_dma_free_iova(struct iommu_domain *domain, dma_addr_t iova,
805+
size_t size, struct iommu_iotlb_gather *gather)
821806
{
822-
struct iova_domain *iovad = &cookie->iovad;
807+
struct iova_domain *iovad = &domain->iova_cookie->iovad;
823808

824809
/* The MSI case is only ever cleaning up its most recent allocation */
825-
if (cookie->type == IOMMU_DMA_MSI_COOKIE)
826-
cookie->msi_iova -= size;
810+
if (domain->cookie_type == IOMMU_COOKIE_DMA_MSI)
811+
domain->msi_cookie->msi_iova -= size;
827812
else if (gather && gather->queued)
828-
queue_iova(cookie, iova_pfn(iovad, iova),
813+
queue_iova(domain->iova_cookie, iova_pfn(iovad, iova),
829814
size >> iova_shift(iovad),
830815
&gather->freelist);
831816
else
@@ -853,7 +838,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
853838

854839
if (!iotlb_gather.queued)
855840
iommu_iotlb_sync(domain, &iotlb_gather);
856-
iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
841+
iommu_dma_free_iova(domain, dma_addr, size, &iotlb_gather);
857842
}
858843

859844
static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -881,7 +866,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
881866
return DMA_MAPPING_ERROR;
882867

883868
if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) {
884-
iommu_dma_free_iova(cookie, iova, size, NULL);
869+
iommu_dma_free_iova(domain, iova, size, NULL);
885870
return DMA_MAPPING_ERROR;
886871
}
887872
return iova + iova_off;
@@ -1018,7 +1003,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
10181003
out_free_sg:
10191004
sg_free_table(sgt);
10201005
out_free_iova:
1021-
iommu_dma_free_iova(cookie, iova, size, NULL);
1006+
iommu_dma_free_iova(domain, iova, size, NULL);
10221007
out_free_pages:
10231008
__iommu_dma_free_pages(pages, count);
10241009
return NULL;
@@ -1495,7 +1480,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
14951480
return __finalise_sg(dev, sg, nents, iova);
14961481

14971482
out_free_iova:
1498-
iommu_dma_free_iova(cookie, iova, iova_len, NULL);
1483+
iommu_dma_free_iova(domain, iova, iova_len, NULL);
14991484
out_restore_sg:
15001485
__invalidate_sg(sg, nents);
15011486
out:
@@ -1773,17 +1758,47 @@ void iommu_setup_dma_ops(struct device *dev)
17731758
dev->dma_iommu = false;
17741759
}
17751760

1761+
static bool has_msi_cookie(const struct iommu_domain *domain)
1762+
{
1763+
return domain && (domain->cookie_type == IOMMU_COOKIE_DMA_IOVA ||
1764+
domain->cookie_type == IOMMU_COOKIE_DMA_MSI);
1765+
}
1766+
1767+
static size_t cookie_msi_granule(const struct iommu_domain *domain)
1768+
{
1769+
switch (domain->cookie_type) {
1770+
case IOMMU_COOKIE_DMA_IOVA:
1771+
return domain->iova_cookie->iovad.granule;
1772+
case IOMMU_COOKIE_DMA_MSI:
1773+
return PAGE_SIZE;
1774+
default:
1775+
unreachable();
1776+
};
1777+
}
1778+
1779+
static struct list_head *cookie_msi_pages(const struct iommu_domain *domain)
1780+
{
1781+
switch (domain->cookie_type) {
1782+
case IOMMU_COOKIE_DMA_IOVA:
1783+
return &domain->iova_cookie->msi_page_list;
1784+
case IOMMU_COOKIE_DMA_MSI:
1785+
return &domain->msi_cookie->msi_page_list;
1786+
default:
1787+
unreachable();
1788+
};
1789+
}
1790+
17761791
static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
17771792
phys_addr_t msi_addr, struct iommu_domain *domain)
17781793
{
1779-
struct iommu_dma_cookie *cookie = domain->iova_cookie;
1794+
struct list_head *msi_page_list = cookie_msi_pages(domain);
17801795
struct iommu_dma_msi_page *msi_page;
17811796
dma_addr_t iova;
17821797
int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1783-
size_t size = cookie_msi_granule(cookie);
1798+
size_t size = cookie_msi_granule(domain);
17841799

17851800
msi_addr &= ~(phys_addr_t)(size - 1);
1786-
list_for_each_entry(msi_page, &cookie->msi_page_list, list)
1801+
list_for_each_entry(msi_page, msi_page_list, list)
17871802
if (msi_page->phys == msi_addr)
17881803
return msi_page;
17891804

@@ -1801,11 +1816,11 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
18011816
INIT_LIST_HEAD(&msi_page->list);
18021817
msi_page->phys = msi_addr;
18031818
msi_page->iova = iova;
1804-
list_add(&msi_page->list, &cookie->msi_page_list);
1819+
list_add(&msi_page->list, msi_page_list);
18051820
return msi_page;
18061821

18071822
out_free_iova:
1808-
iommu_dma_free_iova(cookie, iova, size, NULL);
1823+
iommu_dma_free_iova(domain, iova, size, NULL);
18091824
out_free_page:
18101825
kfree(msi_page);
18111826
return NULL;
@@ -1817,7 +1832,7 @@ static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
18171832
struct device *dev = msi_desc_to_dev(desc);
18181833
const struct iommu_dma_msi_page *msi_page;
18191834

1820-
if (!domain->iova_cookie) {
1835+
if (!has_msi_cookie(domain)) {
18211836
msi_desc_set_iommu_msi_iova(desc, 0, 0);
18221837
return 0;
18231838
}
@@ -1827,9 +1842,8 @@ static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
18271842
if (!msi_page)
18281843
return -ENOMEM;
18291844

1830-
msi_desc_set_iommu_msi_iova(
1831-
desc, msi_page->iova,
1832-
ilog2(cookie_msi_granule(domain->iova_cookie)));
1845+
msi_desc_set_iommu_msi_iova(desc, msi_page->iova,
1846+
ilog2(cookie_msi_granule(domain)));
18331847
return 0;
18341848
}
18351849

drivers/iommu/dma-iommu.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ void iommu_setup_dma_ops(struct device *dev);
1313

1414
int iommu_get_dma_cookie(struct iommu_domain *domain);
1515
void iommu_put_dma_cookie(struct iommu_domain *domain);
16+
void iommu_put_msi_cookie(struct iommu_domain *domain);
1617

1718
int iommu_dma_init_fq(struct iommu_domain *domain);
1819

@@ -40,6 +41,10 @@ static inline void iommu_put_dma_cookie(struct iommu_domain *domain)
4041
{
4142
}
4243

44+
static inline void iommu_put_msi_cookie(struct iommu_domain *domain)
45+
{
46+
}
47+
4348
static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
4449
{
4550
}

drivers/iommu/iommu-sva.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
310310
}
311311

312312
domain->type = IOMMU_DOMAIN_SVA;
313+
domain->cookie_type = IOMMU_COOKIE_SVA;
313314
mmgrab(mm);
314315
domain->mm = mm;
315316
domain->owner = ops;

0 commit comments

Comments
 (0)