Skip to content

Commit

Permalink
Merge branch 'iommu/largepages' into amd-iommu/2.6.35
Browse files Browse the repository at this point in the history 
Conflicts:
	arch/x86/kernel/amd_iommu.c
  • Loading branch information
Joerg Roedel committed May 11, 2010
2 parents a523572 + 12c7389 commit 795e74f
Show file tree
Hide file tree
Showing 6 changed files with 304 additions and 129 deletions.
34 changes: 34 additions & 0 deletions arch/x86/include/asm/amd_iommu_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,40 @@
(~((1ULL << (12 + ((lvl) * 9))) - 1)))
#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr))

/*
* Returns the page table level to use for a given page size
* Pagesize is expected to be a power-of-two
*/
#define PAGE_SIZE_LEVEL(pagesize) \
((__ffs(pagesize) - 12) / 9)
/*
* Returns the number of ptes to use for a given page size
* Pagesize is expected to be a power-of-two
*/
#define PAGE_SIZE_PTE_COUNT(pagesize) \
(1ULL << ((__ffs(pagesize) - 12) % 9))

/*
* Aligns a given io-virtual address to a given page size
* Pagesize is expected to be a power-of-two
*/
#define PAGE_SIZE_ALIGN(address, pagesize) \
((address) & ~((pagesize) - 1))
/*
* Creates an IOMMU PTE for an address an a given pagesize
* The PTE has no permission bits set
* Pagesize is expected to be a power-of-two larger than 4096
*/
#define PAGE_SIZE_PTE(address, pagesize) \
(((address) | ((pagesize) - 1)) & \
(~(pagesize >> 1)) & PM_ADDR_MASK)

/*
* Takes a PTE value with mode=0x07 and returns the page size it maps
*/
#define PTE_PAGE_SIZE(pte) \
(1ULL << (1 + ffz(((pte) | 0xfffULL))))

#define IOMMU_PTE_P (1ULL << 0)
#define IOMMU_PTE_TV (1ULL << 1)
#define IOMMU_PTE_U (1ULL << 59)
Expand Down
197 changes: 126 additions & 71 deletions arch/x86/kernel/amd_iommu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,18 +731,22 @@ static bool increase_address_space(struct protection_domain *domain,

static u64 *alloc_pte(struct protection_domain *domain,
unsigned long address,
int end_lvl,
unsigned long page_size,
u64 **pte_page,
gfp_t gfp)
{
int level, end_lvl;
u64 *pte, *page;
int level;

BUG_ON(!is_power_of_2(page_size));

while (address > PM_LEVEL_SIZE(domain->mode))
increase_address_space(domain, gfp);

level = domain->mode - 1;
pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
level = domain->mode - 1;
pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
address = PAGE_SIZE_ALIGN(address, page_size);
end_lvl = PAGE_SIZE_LEVEL(page_size);

while (level > end_lvl) {
if (!IOMMU_PTE_PRESENT(*pte)) {
Expand All @@ -752,6 +756,10 @@ static u64 *alloc_pte(struct protection_domain *domain,
*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
}

/* No level skipping support yet */
if (PM_PTE_LEVEL(*pte) != level)
return NULL;

level -= 1;

pte = IOMMU_PTE_PAGE(*pte);
Expand All @@ -769,28 +777,47 @@ static u64 *alloc_pte(struct protection_domain *domain,
* This function checks if there is a PTE for a given dma address. If
* there is one, it returns the pointer to it.
*/
static u64 *fetch_pte(struct protection_domain *domain,
unsigned long address, int map_size)
static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
{
int level;
u64 *pte;

level = domain->mode - 1;
pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
if (address > PM_LEVEL_SIZE(domain->mode))
return NULL;

level = domain->mode - 1;
pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];

while (level > map_size) {
while (level > 0) {

/* Not Present */
if (!IOMMU_PTE_PRESENT(*pte))
return NULL;

/* Large PTE */
if (PM_PTE_LEVEL(*pte) == 0x07) {
unsigned long pte_mask, __pte;

/*
* If we have a series of large PTEs, make
* sure to return a pointer to the first one.
*/
pte_mask = PTE_PAGE_SIZE(*pte);
pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
__pte = ((unsigned long)pte) & pte_mask;

return (u64 *)__pte;
}

/* No level skipping support yet */
if (PM_PTE_LEVEL(*pte) != level)
return NULL;

level -= 1;

/* Walk to the next level */
pte = IOMMU_PTE_PAGE(*pte);
pte = &pte[PM_LEVEL_INDEX(level, address)];

if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) {
pte = NULL;
break;
}
}

return pte;
Expand All @@ -807,44 +834,84 @@ static int iommu_map_page(struct protection_domain *dom,
unsigned long bus_addr,
unsigned long phys_addr,
int prot,
int map_size)
unsigned long page_size)
{
u64 __pte, *pte;

bus_addr = PAGE_ALIGN(bus_addr);
phys_addr = PAGE_ALIGN(phys_addr);

BUG_ON(!PM_ALIGNED(map_size, bus_addr));
BUG_ON(!PM_ALIGNED(map_size, phys_addr));
int i, count;

if (!(prot & IOMMU_PROT_MASK))
return -EINVAL;

pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL);
bus_addr = PAGE_ALIGN(bus_addr);
phys_addr = PAGE_ALIGN(phys_addr);
count = PAGE_SIZE_PTE_COUNT(page_size);
pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);

for (i = 0; i < count; ++i)
if (IOMMU_PTE_PRESENT(pte[i]))
return -EBUSY;

if (IOMMU_PTE_PRESENT(*pte))
return -EBUSY;
if (page_size > PAGE_SIZE) {
__pte = PAGE_SIZE_PTE(phys_addr, page_size);
__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
} else
__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;

__pte = phys_addr | IOMMU_PTE_P;
if (prot & IOMMU_PROT_IR)
__pte |= IOMMU_PTE_IR;
if (prot & IOMMU_PROT_IW)
__pte |= IOMMU_PTE_IW;

*pte = __pte;
for (i = 0; i < count; ++i)
pte[i] = __pte;

update_domain(dom);

return 0;
}

static void iommu_unmap_page(struct protection_domain *dom,
unsigned long bus_addr, int map_size)
static unsigned long iommu_unmap_page(struct protection_domain *dom,
unsigned long bus_addr,
unsigned long page_size)
{
u64 *pte = fetch_pte(dom, bus_addr, map_size);
unsigned long long unmap_size, unmapped;
u64 *pte;

BUG_ON(!is_power_of_2(page_size));

unmapped = 0;

if (pte)
*pte = 0;
while (unmapped < page_size) {

pte = fetch_pte(dom, bus_addr);

if (!pte) {
/*
* No PTE for this address
* move forward in 4kb steps
*/
unmap_size = PAGE_SIZE;
} else if (PM_PTE_LEVEL(*pte) == 0) {
/* 4kb PTE found for this address */
unmap_size = PAGE_SIZE;
*pte = 0ULL;
} else {
int count, i;

/* Large PTE found which maps this address */
unmap_size = PTE_PAGE_SIZE(*pte);
count = PAGE_SIZE_PTE_COUNT(unmap_size);
for (i = 0; i < count; i++)
pte[i] = 0ULL;
}

bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
unmapped += unmap_size;
}

BUG_ON(!is_power_of_2(unmapped));

return unmapped;
}

/*
Expand Down Expand Up @@ -878,7 +945,7 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
for (addr = e->address_start; addr < e->address_end;
addr += PAGE_SIZE) {
ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
PM_MAP_4k);
PAGE_SIZE);
if (ret)
return ret;
/*
Expand Down Expand Up @@ -1006,7 +1073,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
u64 *pte, *pte_page;

for (i = 0; i < num_ptes; ++i) {
pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k,
pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
&pte_page, gfp);
if (!pte)
goto out_free;
Expand Down Expand Up @@ -1042,7 +1109,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
for (i = dma_dom->aperture[index]->offset;
i < dma_dom->aperture_size;
i += PAGE_SIZE) {
u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k);
u64 *pte = fetch_pte(&dma_dom->domain, i);
if (!pte || !IOMMU_PTE_PRESENT(*pte))
continue;

Expand Down Expand Up @@ -1712,7 +1779,7 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom,

pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
if (!pte) {
pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page,
pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
GFP_ATOMIC);
aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
} else
Expand Down Expand Up @@ -2439,12 +2506,11 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
return ret;
}

static int amd_iommu_map_range(struct iommu_domain *dom,
unsigned long iova, phys_addr_t paddr,
size_t size, int iommu_prot)
static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
phys_addr_t paddr, int gfp_order, int iommu_prot)
{
unsigned long page_size = 0x1000UL << gfp_order;
struct protection_domain *domain = dom->priv;
unsigned long i, npages = iommu_num_pages(paddr, size, PAGE_SIZE);
int prot = 0;
int ret;

Expand All @@ -2453,61 +2519,50 @@ static int amd_iommu_map_range(struct iommu_domain *dom,
if (iommu_prot & IOMMU_WRITE)
prot |= IOMMU_PROT_IW;

iova &= PAGE_MASK;
paddr &= PAGE_MASK;

mutex_lock(&domain->api_lock);

for (i = 0; i < npages; ++i) {
ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k);
if (ret)
return ret;

iova += PAGE_SIZE;
paddr += PAGE_SIZE;
}

ret = iommu_map_page(domain, iova, paddr, prot, page_size);
mutex_unlock(&domain->api_lock);

return 0;
return ret;
}

static void amd_iommu_unmap_range(struct iommu_domain *dom,
unsigned long iova, size_t size)
static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
int gfp_order)
{

struct protection_domain *domain = dom->priv;
unsigned long i, npages = iommu_num_pages(iova, size, PAGE_SIZE);
unsigned long page_size, unmap_size;

iova &= PAGE_MASK;
page_size = 0x1000UL << gfp_order;

mutex_lock(&domain->api_lock);

for (i = 0; i < npages; ++i) {
iommu_unmap_page(domain, iova, PM_MAP_4k);
iova += PAGE_SIZE;
}
unmap_size = iommu_unmap_page(domain, iova, page_size);
mutex_unlock(&domain->api_lock);

iommu_flush_tlb_pde(domain);

mutex_unlock(&domain->api_lock);
return get_order(unmap_size);
}

static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
unsigned long iova)
{
struct protection_domain *domain = dom->priv;
unsigned long offset = iova & ~PAGE_MASK;
unsigned long offset_mask;
phys_addr_t paddr;
u64 *pte;
u64 *pte, __pte;

pte = fetch_pte(domain, iova, PM_MAP_4k);
pte = fetch_pte(domain, iova);

if (!pte || !IOMMU_PTE_PRESENT(*pte))
return 0;

paddr = *pte & IOMMU_PAGE_MASK;
paddr |= offset;
if (PM_PTE_LEVEL(*pte) == 0)
offset_mask = PAGE_SIZE - 1;
else
offset_mask = PTE_PAGE_SIZE(*pte) - 1;

__pte = *pte & PM_ADDR_MASK;
paddr = (__pte & ~offset_mask) | (iova & offset_mask);

return paddr;
}
Expand All @@ -2523,8 +2578,8 @@ static struct iommu_ops amd_iommu_ops = {
.domain_destroy = amd_iommu_domain_destroy,
.attach_dev = amd_iommu_attach_device,
.detach_dev = amd_iommu_detach_device,
.map = amd_iommu_map_range,
.unmap = amd_iommu_unmap_range,
.map = amd_iommu_map,
.unmap = amd_iommu_unmap,
.iova_to_phys = amd_iommu_iova_to_phys,
.domain_has_cap = amd_iommu_domain_has_cap,
};
Expand Down
Loading

0 comments on commit 795e74f

Please sign in to comment.