From e81e99bacc9f9347bda7808a949c1ce9fcc2bbf4 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:59 +0900 Subject: swiotlb: Support aligned swiotlb buffers Add an argument to swiotlb_tbl_map_single that specifies the desired alignment of the allocated buffer. This is used by dma-iommu to ensure the buffer is aligned to the iova granule size when using swiotlb with untrusted sub-granule mappings. This addresses an issue where adjacent slots could be exposed to the untrusted device if IO_TLB_SIZE < iova granule < PAGE_SIZE. Signed-off-by: David Stevens Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210929023300.335969-7-stevensd@google.com Signed-off-by: Joerg Roedel --- include/linux/swiotlb.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index b0cb2a9973f4..569272871375 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -45,7 +45,8 @@ extern void __init swiotlb_update_mem_attributes(void); phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, size_t mapping_size, size_t alloc_size, - enum dma_data_direction dir, unsigned long attrs); + unsigned int alloc_aligned_mask, enum dma_data_direction dir, + unsigned long attrs); extern void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, -- cgit v1.2.3 From 914ff7719e8adaf87131dd7cd0a3520afcf89516 Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Thu, 14 Oct 2021 13:38:32 +0800 Subject: iommu/vt-d: Dump DMAR translation structure when DMA fault occurs When the dmar translation fault happens, the kernel prints a single line fault reason with corresponding hexadecimal code defined in the Intel VT-d specification. Currently, when user wants to debug the translation fault in detail, debugfs is used for dumping the dmar_translation_struct, which is not available when the kernel failed to boot. Dump the DMAR translation structure, pagewalk the IO page table and print the page table entry when the fault happens. This takes effect only when CONFIG_DMAR_DEBUG is enabled. Signed-off-by: Kyung Min Park Link: https://lore.kernel.org/r/20210815203845.31287-1-kyung.min.park@intel.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/Kconfig | 4 ++ drivers/iommu/intel/dmar.c | 10 +++- drivers/iommu/intel/iommu.c | 113 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/dmar.h | 8 ++++ 4 files changed, 133 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 0ddb77115be7..247d0f2d5fdf 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -6,6 +6,9 @@ config DMAR_TABLE config DMAR_PERF bool +config DMAR_DEBUG + bool + config INTEL_IOMMU bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64) @@ -31,6 +34,7 @@ config INTEL_IOMMU_DEBUGFS bool "Export Intel IOMMU internals in Debugfs" depends on IOMMU_DEBUGFS select DMAR_PERF + select DMAR_DEBUG help !!!WARNING!!! diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index b7708b93f3fa..915bff76fe96 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1941,12 +1941,16 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, reason = dmar_get_fault_reason(fault_reason, &fault_type); - if (fault_type == INTR_REMAP) + if (fault_type == INTR_REMAP) { pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n", source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr >> 48, fault_reason, reason); - else if (pasid == INVALID_IOASID) + + return 0; + } + + if (pasid == INVALID_IOASID) pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", type ? "DMA Read" : "DMA Write", source_id >> 8, PCI_SLOT(source_id & 0xFF), @@ -1959,6 +1963,8 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason); + dmar_fault_dump_ptes(iommu, source_id, addr, pasid); + return 0; } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 9a356075d345..abb2599998b1 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -156,6 +156,8 @@ static struct intel_iommu **g_iommus; static void __init check_tylersburg_isoch(void); static int rwbf_quirk; +static inline struct device_domain_info * +dmar_search_domain_by_dev_info(int segment, int bus, int devfn); /* * set to 1 to panic kernel if can't successfully enable VT-d @@ -996,6 +998,117 @@ out: spin_unlock_irqrestore(&iommu->lock, flags); } +#ifdef CONFIG_DMAR_DEBUG +static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn, u8 bus, u8 devfn) +{ + struct device_domain_info *info; + struct dma_pte *parent, *pte; + struct dmar_domain *domain; + int offset, level; + + info = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn); + if (!info || !info->domain) { + pr_info("device [%02x:%02x.%d] not probed\n", + bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + return; + } + + domain = info->domain; + level = agaw_to_level(domain->agaw); + parent = domain->pgd; + if (!parent) { + pr_info("no page table setup\n"); + return; + } + + while (1) { + offset = pfn_level_offset(pfn, level); + pte = &parent[offset]; + if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) { + pr_info("PTE not present at level %d\n", level); + break; + } + + pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val); + + if (level == 1) + break; + + parent = phys_to_virt(dma_pte_addr(pte)); + level--; + } +} + +void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, + unsigned long long addr, u32 pasid) +{ + struct pasid_dir_entry *dir, *pde; + struct pasid_entry *entries, *pte; + struct context_entry *ctx_entry; + struct root_entry *rt_entry; + u8 devfn = source_id & 0xff; + u8 bus = source_id >> 8; + int i, dir_index, index; + + pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr); + + /* root entry dump */ + rt_entry = &iommu->root_entry[bus]; + if (!rt_entry) { + pr_info("root table entry is not present\n"); + return; + } + + if (sm_supported(iommu)) + pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n", + rt_entry->hi, rt_entry->lo); + else + pr_info("root entry: 0x%016llx", rt_entry->lo); + + /* context entry dump */ + ctx_entry = iommu_context_addr(iommu, bus, devfn, 0); + if (!ctx_entry) { + pr_info("context table entry is not present\n"); + return; + } + + pr_info("context entry: hi 0x%016llx, low 0x%016llx\n", + ctx_entry->hi, ctx_entry->lo); + + /* legacy mode does not require PASID entries */ + if (!sm_supported(iommu)) + goto pgtable_walk; + + /* get the pointer to pasid directory entry */ + dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK); + if (!dir) { + pr_info("pasid directory entry is not present\n"); + return; + } + /* For request-without-pasid, get the pasid from context entry */ + if (intel_iommu_sm && pasid == INVALID_IOASID) + pasid = PASID_RID2PASID; + + dir_index = pasid >> PASID_PDE_SHIFT; + pde = &dir[dir_index]; + pr_info("pasid dir entry: 0x%016llx\n", pde->val); + + /* get the pointer to the pasid table entry */ + entries = get_pasid_table_from_pde(pde); + if (!entries) { + pr_info("pasid table entry is not present\n"); + return; + } + index = pasid & PASID_PTE_MASK; + pte = &entries[index]; + for (i = 0; i < ARRAY_SIZE(pte->val); i++) + pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]); + +pgtable_walk: + pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn); +} +#endif + static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, unsigned long pfn, int *target_level) { diff --git a/include/linux/dmar.h b/include/linux/dmar.h index e04436a7ff27..45e903d84733 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -131,6 +131,14 @@ static inline int dmar_res_noop(struct acpi_dmar_header *hdr, void *arg) return 0; } +#ifdef CONFIG_DMAR_DEBUG +void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, + unsigned long long addr, u32 pasid); +#else +static inline void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, + unsigned long long addr, u32 pasid) {} +#endif + #ifdef CONFIG_INTEL_IOMMU extern int iommu_detected, no_iommu; extern int intel_iommu_init(void); -- cgit v1.2.3 From b34380a6d767c54480a937951e6189a7f9699443 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Oct 2021 13:38:33 +0800 Subject: iommu/vt-d: Remove duplicate identity domain flag The iommu_domain data structure already has the "type" field to keep the type of a domain. It's unnecessary to have the DOMAIN_FLAG_STATIC_IDENTITY flag in the vt-d implementation. This cleans it up with no functionality change. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20210926114535.923263-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20211014053839.727419-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 9 ++++----- include/linux/intel-iommu.h | 3 --- 2 files changed, 4 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index abb2599998b1..ec9ceb2dcf57 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -528,7 +528,7 @@ static inline void free_devinfo_mem(void *vaddr) static inline int domain_type_is_si(struct dmar_domain *domain) { - return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY; + return domain->domain.type == IOMMU_DOMAIN_IDENTITY; } static inline bool domain_use_first_level(struct dmar_domain *domain) @@ -1996,7 +1996,7 @@ static bool first_level_by_default(void) return scalable_mode_support() && intel_cap_flts_sanity(); } -static struct dmar_domain *alloc_domain(int flags) +static struct dmar_domain *alloc_domain(unsigned int type) { struct dmar_domain *domain; @@ -2006,7 +2006,6 @@ static struct dmar_domain *alloc_domain(int flags) memset(domain, 0, sizeof(*domain)); domain->nid = NUMA_NO_NODE; - domain->flags = flags; if (first_level_by_default()) domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL; domain->has_iotlb_device = false; @@ -2825,7 +2824,7 @@ static int __init si_domain_init(int hw) struct device *dev; int i, nid, ret; - si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY); + si_domain = alloc_domain(IOMMU_DOMAIN_IDENTITY); if (!si_domain) return -EFAULT; @@ -4634,7 +4633,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) case IOMMU_DOMAIN_DMA: case IOMMU_DOMAIN_DMA_FQ: case IOMMU_DOMAIN_UNMANAGED: - dmar_domain = alloc_domain(0); + dmar_domain = alloc_domain(type); if (!dmar_domain) { pr_err("Can't allocate dmar_domain\n"); return NULL; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 05a65eb155f7..65b15af3cf1a 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -517,9 +517,6 @@ struct context_entry { u64 hi; }; -/* si_domain contains mulitple devices */ -#define DOMAIN_FLAG_STATIC_IDENTITY BIT(0) - /* * When VT-d works in the scalable mode, it allows DMA translation to * happen through either first level or second level page table. This -- cgit v1.2.3 From 37c8041a818dec2cea22ea48a3f90d512a9e1fcd Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Thu, 14 Oct 2021 13:38:38 +0800 Subject: iommu/vt-d: Convert the return type of first_pte_in_page to bool The first_pte_in_page() returns true or false, so let's convert its return type to bool. In addition, use 'IS_ALIGNED' to make the code more readable and neater. Signed-off-by: Longpeng(Mike) Link: https://lore.kernel.org/r/20211008000433.1115-1-longpeng2@huawei.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/intel-iommu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 65b15af3cf1a..52481625838c 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -705,9 +705,9 @@ static inline bool dma_pte_superpage(struct dma_pte *pte) return (pte->val & DMA_PTE_LARGE_PAGE); } -static inline int first_pte_in_page(struct dma_pte *pte) +static inline bool first_pte_in_page(struct dma_pte *pte) { - return !((unsigned long)pte & ~VTD_PAGE_MASK); + return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); } extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); -- cgit v1.2.3 From 9906b9352a35427508d974db3a496eb3c49e2010 Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Thu, 14 Oct 2021 13:38:39 +0800 Subject: iommu/vt-d: Avoid duplicate removing in __domain_mapping() The __domain_mapping() always removes the pages in the range from 'iov_pfn' to 'end_pfn', but the 'end_pfn' is always the last pfn of the range that the caller wants to map. This would introduce too many duplicated removing and leads the map operation take too long, for example: Map iova=0x100000,nr_pages=0x7d61800 iov_pfn: 0x100000, end_pfn: 0x7e617ff iov_pfn: 0x140000, end_pfn: 0x7e617ff iov_pfn: 0x180000, end_pfn: 0x7e617ff iov_pfn: 0x1c0000, end_pfn: 0x7e617ff iov_pfn: 0x200000, end_pfn: 0x7e617ff ... it takes about 50ms in total. We can reduce the cost by recalculate the 'end_pfn' and limit it to the boundary of the end of this pte page. Map iova=0x100000,nr_pages=0x7d61800 iov_pfn: 0x100000, end_pfn: 0x13ffff iov_pfn: 0x140000, end_pfn: 0x17ffff iov_pfn: 0x180000, end_pfn: 0x1bffff iov_pfn: 0x1c0000, end_pfn: 0x1fffff iov_pfn: 0x200000, end_pfn: 0x23ffff ... it only need 9ms now. This also removes a meaningless BUG_ON() in __domain_mapping(). Signed-off-by: Longpeng(Mike) Tested-by: Liujunjie Link: https://lore.kernel.org/r/20211008000433.1115-1-longpeng2@huawei.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 11 ++++++----- include/linux/intel-iommu.h | 6 ++++++ 2 files changed, 12 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 16a35669a9d0..0bde0c8b4126 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2479,12 +2479,17 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -ENOMEM; first_pte = pte; + lvl_pages = lvl_to_nr_pages(largepage_lvl); + /* It is large page*/ if (largepage_lvl > 1) { unsigned long end_pfn; + unsigned long pages_to_remove; pteval |= DMA_PTE_LARGE_PAGE; - end_pfn = ((iov_pfn + nr_pages) & level_mask(largepage_lvl)) - 1; + pages_to_remove = min_t(unsigned long, nr_pages, + nr_pte_to_next_page(pte) * lvl_pages); + end_pfn = iov_pfn + pages_to_remove - 1; switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl); } else { pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE; @@ -2506,10 +2511,6 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, WARN_ON(1); } - lvl_pages = lvl_to_nr_pages(largepage_lvl); - - BUG_ON(nr_pages < lvl_pages); - nr_pages -= lvl_pages; iov_pfn += lvl_pages; phys_pfn += lvl_pages; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 52481625838c..69230fd695ea 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -710,6 +710,12 @@ static inline bool first_pte_in_page(struct dma_pte *pte) return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); } +static inline int nr_pte_to_next_page(struct dma_pte *pte) +{ + return first_pte_in_page(pte) ? BIT_ULL(VTD_STRIDE_SHIFT) : + (struct dma_pte *)ALIGN((unsigned long)pte, VTD_PAGE_SIZE) - pte; +} + extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); extern int dmar_find_matched_atsr_unit(struct pci_dev *dev); -- cgit v1.2.3