From bccc37a8a2fb002a302a526656c56a793a708670 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 30 May 2023 10:11:37 -0400 Subject: [PATCH] iommu/amd: Improving Interrupt Remapping Table Invalidation Invalidating Interrupt Remapping Table (IRT) requires, the AMD IOMMU driver to issue INVALIDATE_INTERRUPT_TABLE and COMPLETION_WAIT commands. Currently, the driver issues the two commands separately, which requires calling raw_spin_lock_irqsave() twice. In addition, the COMPLETION_WAIT could potentially be interleaved with other commands causing delay of the COMPLETION_WAIT command. Therefore, combine issuing of the two commands in one spin-lock, and changing struct amd_iommu.cmd_sem_val to use atomic64 to minimize locking. Reviewed-by: Jerry Snitselaar Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20230530141137.14376-6-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/init.c | 2 +- drivers/iommu/amd/iommu.c | 27 ++++++++++++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 8eeea1f25e6978..5a4e04404cfd64 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -752,7 +752,7 @@ struct amd_iommu { u32 flags; volatile u64 *cmd_sem; - u64 cmd_sem_val; + atomic64_t cmd_sem_val; #ifdef CONFIG_AMD_IOMMU_DEBUGFS /* DebugFS Info */ diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 418da641ee3da8..4c68dc382286c8 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1733,7 +1733,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, iommu->pci_seg = pci_seg; raw_spin_lock_init(&iommu->lock); - iommu->cmd_sem_val = 0; + atomic64_set(&iommu->cmd_sem_val, 0); /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 8fdd6ebf87118c..e47c8c52070801 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1175,11 +1175,11 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (!iommu->need_sync) return 0; - raw_spin_lock_irqsave(&iommu->lock, flags); - - data = ++iommu->cmd_sem_val; + data = atomic64_add_return(1, &iommu->cmd_sem_val); build_completion_wait(&cmd, iommu, data); + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __iommu_queue_command_sync(iommu, &cmd, false); if (ret) goto out_unlock; @@ -1277,11 +1277,28 @@ static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) { + int ret; + u64 data; + unsigned long flags; + struct iommu_cmd cmd, cmd2; + if (iommu->irtcachedis_enabled) return; - iommu_flush_irt(iommu, devid); - iommu_completion_wait(iommu); + build_inv_irt(&cmd, devid); + data = atomic64_add_return(1, &iommu->cmd_sem_val); + build_completion_wait(&cmd2, iommu, data); + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __iommu_queue_command_sync(iommu, &cmd, true); + if (ret) + goto out; + ret = __iommu_queue_command_sync(iommu, &cmd2, false); + if (ret) + goto out; + wait_on_sem(iommu, data); +out: + raw_spin_unlock_irqrestore(&iommu->lock, flags); } void iommu_flush_all_caches(struct amd_iommu *iommu)