diff options
| -rw-r--r-- | usr/src/uts/i86pc/io/intel_iommu.c | 32 | ||||
| -rw-r--r-- | usr/src/uts/i86pc/io/iommu_rscs.c | 159 | ||||
| -rw-r--r-- | usr/src/uts/i86pc/sys/intel_iommu.h | 5 | ||||
| -rw-r--r-- | usr/src/uts/i86pc/sys/iommu_rscs.h | 20 | ||||
| -rw-r--r-- | usr/src/uts/i86pc/sys/machsystm.h | 3 | ||||
| -rw-r--r-- | usr/src/uts/i86pc/vm/htable.c | 27 | ||||
| -rw-r--r-- | usr/src/uts/i86pc/vm/vm_machdep.c | 43 |
7 files changed, 176 insertions, 113 deletions
diff --git a/usr/src/uts/i86pc/io/intel_iommu.c b/usr/src/uts/i86pc/io/intel_iommu.c index c1e21c6e33..198e7f70e1 100644 --- a/usr/src/uts/i86pc/io/intel_iommu.c +++ b/usr/src/uts/i86pc/io/intel_iommu.c @@ -274,18 +274,17 @@ iommu_page_init(void) static paddr_t iommu_get_page(intel_iommu_state_t *iommu, int kmflag) { - paddr_t paddr; + iommu_pghdl_t *pghdl; caddr_t vaddr; - paddr = iommu_page_alloc(kmflag); - vaddr = iommu_page_map(paddr); + pghdl = iommu_page_alloc(iommu, kmflag); + vaddr = pghdl->vaddr; bzero(vaddr, IOMMU_PAGE_SIZE); iommu->iu_dmar_ops->do_clflush(vaddr, IOMMU_PAGE_SIZE); - iommu_page_unmap(vaddr); page_num++; - return (paddr); + return (pghdl->paddr); } /* @@ -293,9 +292,9 @@ iommu_get_page(intel_iommu_state_t *iommu, int kmflag) * free the iommu page allocated with iommu_get_page */ static void -iommu_free_page(paddr_t paddr) +iommu_free_page(intel_iommu_state_t *iommu, paddr_t paddr) { - iommu_page_free(paddr); + iommu_page_free(iommu, paddr); page_num--; } @@ -339,7 +338,7 @@ calculate_agaw(int gaw) static void destroy_iommu_state(intel_iommu_state_t *iommu) { - iommu_free_page(iommu->iu_root_entry_paddr); + iommu_free_page(iommu, iommu->iu_root_entry_paddr); iommu_rscs_fini(&(iommu->iu_domain_id_hdl)); mutex_destroy(&(iommu->iu_reg_lock)); mutex_destroy(&(iommu->iu_root_context_lock)); @@ -1827,14 +1826,15 @@ iommu_domain_init(dmar_domain_state_t *domain) /* * create the first level page table */ - domain->dm_page_table_paddr = - iommu_get_page(domain->dm_iommu, KM_SLEEP); + domain->dm_page_table_paddr = iommu_get_page(domain->dm_iommu, + KM_SLEEP); /* * init the CPU available page tables */ domain->dm_pt_tree.vp = kmem_zalloc(IOMMU_PAGE_SIZE << 1, KM_SLEEP); - domain->dm_pt_tree.pp = iommu_page_map(domain->dm_page_table_paddr); + domain->dm_pt_tree.pp = iommu_get_vaddr(domain->dm_iommu, + domain->dm_page_table_paddr); domain->dm_identity = B_FALSE; /* @@ -1972,7 +1972,7 @@ domain_set_root_context(dmar_domain_state_t *domain, /* * set root entry */ - root = iommu_page_map(iommu->iu_root_entry_paddr); + root = iommu_get_vaddr(iommu, iommu->iu_root_entry_paddr); rce = (iorce_t)root + bus; mutex_enter(&(iommu->iu_root_context_lock)); if (!ROOT_ENTRY_GET_P(rce)) { @@ -1980,10 +1980,10 @@ domain_set_root_context(dmar_domain_state_t *domain, ROOT_ENTRY_SET_P(rce); ROOT_ENTRY_SET_CTP(rce, paddr); iommu->iu_dmar_ops->do_clflush((caddr_t)rce, sizeof (*rce)); - context = iommu_page_map(paddr); + context = iommu_get_vaddr(iommu, paddr); } else { paddr = ROOT_ENTRY_GET_CTP(rce); - context = iommu_page_map(paddr); + context = iommu_get_vaddr(iommu, paddr); } /* set context entry */ @@ -2003,8 +2003,6 @@ domain_set_root_context(dmar_domain_state_t *domain, } mutex_exit(&(iommu->iu_root_context_lock)); - iommu_page_unmap(root); - iommu_page_unmap(context); /* cache mode set, flush context cache */ if (IOMMU_CAP_GET_CM(iommu->iu_capability)) { @@ -2300,7 +2298,7 @@ iommu_setup_level_table(dmar_domain_state_t *domain, domain->dm_iommu->iu_dmar_ops->do_clflush((caddr_t)pte, sizeof (*pte)); vpte->vp = kmem_zalloc(IOMMU_PAGE_SIZE << 1, KM_SLEEP); - vpte->pp = iommu_page_map(child); + vpte->pp = iommu_get_vaddr(domain->dm_iommu, child); } return (vpte); diff --git a/usr/src/uts/i86pc/io/iommu_rscs.c b/usr/src/uts/i86pc/io/iommu_rscs.c index 6122a446a4..318f38481e 100644 --- a/usr/src/uts/i86pc/io/iommu_rscs.c +++ b/usr/src/uts/i86pc/io/iommu_rscs.c @@ -19,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -50,8 +50,28 @@ #include <vm/hat_i86.h> #include <sys/machsystm.h> #include <sys/iommu_rscs.h> - - +#include <sys/intel_iommu.h> + +ddi_dma_attr_t page_dma_attr = { + DMA_ATTR_V0, + 0U, + 0xffffffffU, + 0xffffffffU, + MMU_PAGESIZE, /* page aligned */ + 0x1, + 0x1, + 0xffffffffU, + 0xffffffffU, + 1, + 4, + 0 +}; + +ddi_device_acc_attr_t page_acc_attr = { + DDI_DEVICE_ATTR_V0, + DDI_NEVERSWAP_ACC, + DDI_STRICTORDER_ACC +}; typedef struct iommu_rscs_s { /* @@ -78,87 +98,118 @@ typedef struct iommu_rscs_s { kmutex_t rs_mutex; } iommu_rscs_state_t; +static uint_t +iommu_pghdl_hash_func(paddr_t paddr) +{ + return (paddr % IOMMU_PGHDL_HASH_SIZE); +} /* * iommu_page_alloc() * */ -paddr_t -iommu_page_alloc(int kmflag) +iommu_pghdl_t * +iommu_page_alloc(intel_iommu_state_t *iommu, int kmflag) { - paddr_t paddr; - page_t *pp; + size_t actual_size = 0; + iommu_pghdl_t *pghdl; + caddr_t vaddr; + uint_t idx; ASSERT(kmflag == KM_SLEEP || kmflag == KM_NOSLEEP); - pp = page_get_physical(kmflag); - if (pp == NULL) { - return (NULL); + pghdl = kmem_zalloc(sizeof (*pghdl), kmflag); + if (pghdl == NULL) { + return (0); } - paddr = pa_to_ma((uint64_t)pp->p_pagenum << PAGESHIFT); + if (ddi_dma_alloc_handle(ddi_root_node(), &page_dma_attr, DDI_DMA_SLEEP, + NULL, &pghdl->dma_hdl) != DDI_SUCCESS) { + kmem_free(pghdl, sizeof (*pghdl)); + return (0); + } - return (paddr); -} + if (ddi_dma_mem_alloc(pghdl->dma_hdl, PAGESIZE, &page_acc_attr, + DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED, + (kmflag == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, + NULL, &vaddr, &actual_size, &pghdl->mem_hdl) != DDI_SUCCESS) { + ddi_dma_free_handle(&pghdl->dma_hdl); + kmem_free(pghdl, sizeof (*pghdl)); + return (0); + } + ASSERT(actual_size == PAGESIZE); -/* - * iommu_page_free() - */ -void -iommu_page_free(paddr_t paddr) -{ - page_t *pp; + if (actual_size != PAGESIZE) { + ddi_dma_mem_free(&pghdl->mem_hdl); + ddi_dma_free_handle(&pghdl->dma_hdl); + kmem_free(pghdl, sizeof (*pghdl)); + return (0); - pp = page_numtopp_nolock(ma_to_pa(paddr) >> PAGESHIFT); - page_free_physical(pp); -} + } + pghdl->paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, vaddr)); + + idx = iommu_pghdl_hash_func(pghdl->paddr); + pghdl->next = iommu->iu_pghdl_hash[idx]; + if (pghdl->next) + pghdl->next->prev = pghdl; + iommu->iu_pghdl_hash[idx] = pghdl; + + return (pghdl); +} /* - * iommu_page_map() - * + * iommu_page_free() */ -caddr_t -iommu_page_map(paddr_t addr) +void +iommu_page_free(intel_iommu_state_t *iommu, paddr_t paddr) { - paddr_t paddr; - caddr_t kva; - page_t *pp; - - paddr = ma_to_pa(addr); - - if (kpm_enable) { - kva = hat_kpm_pfn2va((pfn_t)btop(paddr)); - } else { - kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); - if (kva == NULL) { - return (NULL); - } - pp = page_numtopp_nolock(paddr >> PAGESHIFT); - hat_memload(kas.a_hat, kva, pp, - PROT_READ | PROT_WRITE, HAT_LOAD_LOCK); + uint_t idx; + iommu_pghdl_t *pghdl; + + idx = iommu_pghdl_hash_func(paddr); + pghdl = iommu->iu_pghdl_hash[idx]; + while (pghdl && pghdl->paddr != paddr) + continue; + if (pghdl == NULL) { + cmn_err(CE_PANIC, + "Freeing a free IOMMU page: paddr=0x%" PRIx64, + paddr); + /*NOTREACHED*/ } - - return (kva); + if (pghdl->prev == NULL) + iommu->iu_pghdl_hash[idx] = pghdl->next; + else + pghdl->prev->next = pghdl->next; + if (pghdl->next) + pghdl->next->prev = pghdl->prev; + + ddi_dma_mem_free(&pghdl->mem_hdl); + ddi_dma_free_handle(&pghdl->dma_hdl); + kmem_free(pghdl, sizeof (*pghdl)); } - /* - * iommu_page_unmap() - * + * iommu_get_vaddr() */ -void -iommu_page_unmap(caddr_t kva) +caddr_t +iommu_get_vaddr(intel_iommu_state_t *iommu, paddr_t paddr) { - if (!kpm_enable) { - hat_unload(kas.a_hat, kva, PAGESIZE, HAT_UNLOAD_UNLOCK); - vmem_free(heap_arena, kva, PAGESIZE); + uint_t idx; + iommu_pghdl_t *pghdl; + + idx = iommu_pghdl_hash_func(paddr); + pghdl = iommu->iu_pghdl_hash[idx]; + while (pghdl && pghdl->paddr != paddr) + continue; + if (pghdl == NULL) { + return (0); } + return (pghdl->vaddr); } - /* * iommu_rscs_init() * Initialize the resource structure. init() returns a handle to be diff --git a/usr/src/uts/i86pc/sys/intel_iommu.h b/usr/src/uts/i86pc/sys/intel_iommu.h index 9797782f88..3a157a45ed 100644 --- a/usr/src/uts/i86pc/sys/intel_iommu.h +++ b/usr/src/uts/i86pc/sys/intel_iommu.h @@ -413,6 +413,9 @@ typedef struct iotlb_pend_head { struct inv_queue_state; struct intr_remap_tbl_state; +struct iommu_pghdl; + +#define IOMMU_PGHDL_HASH_SIZE (256) /* * struct intel_iommu_state @@ -445,6 +448,7 @@ struct intr_remap_tbl_state; * iu_pend_head - pending iotlb list * iu_inv_queue - invalidation queue state * iu_intr_remap_tbl - interrupt remapping table state + * iu_pghdl_hash - hash of pages allocated for IOMMU internal work. */ typedef struct intel_iommu_state { list_node_t node; @@ -470,6 +474,7 @@ typedef struct intel_iommu_state { iotlb_pend_head_t iu_pend_head; struct inv_queue_state *iu_inv_queue; struct intr_remap_tbl_state *iu_intr_remap_tbl; + struct iommu_pghdl *iu_pghdl_hash[IOMMU_PGHDL_HASH_SIZE]; } intel_iommu_state_t; /* diff --git a/usr/src/uts/i86pc/sys/iommu_rscs.h b/usr/src/uts/i86pc/sys/iommu_rscs.h index 60a073c402..0c8519e3ca 100644 --- a/usr/src/uts/i86pc/sys/iommu_rscs.h +++ b/usr/src/uts/i86pc/sys/iommu_rscs.h @@ -19,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -50,11 +50,21 @@ extern "C" { * iommu_page_unmap() * unmap page out of kva */ -paddr_t iommu_page_alloc(int kmflag); -void iommu_page_free(paddr_t paddr); -caddr_t iommu_page_map(paddr_t paddr); -void iommu_page_unmap(caddr_t kva); +typedef struct iommu_pghdl { + ddi_dma_handle_t dma_hdl; + ddi_acc_handle_t mem_hdl; + paddr_t paddr; + caddr_t vaddr; + struct iommu_pghdl *prev; + struct iommu_pghdl *next; +} iommu_pghdl_t; + +struct intel_iommu_state; + +iommu_pghdl_t *iommu_page_alloc(struct intel_iommu_state *iommu, int kmflag); +void iommu_page_free(struct intel_iommu_state *iommu, paddr_t paddr); +caddr_t iommu_get_vaddr(struct intel_iommu_state *iommu, paddr_t paddr); typedef struct iommu_rscs_s *iommu_rscs_t; diff --git a/usr/src/uts/i86pc/sys/machsystm.h b/usr/src/uts/i86pc/sys/machsystm.h index feebea3f6c..928f0b0d12 100644 --- a/usr/src/uts/i86pc/sys/machsystm.h +++ b/usr/src/uts/i86pc/sys/machsystm.h @@ -138,8 +138,7 @@ extern caddr_t kpm_vbase; struct memlist; extern void memlist_add(uint64_t, uint64_t, struct memlist *, struct memlist **); -extern page_t *page_get_physical(int flags); -extern void page_free_physical(page_t *); +extern page_t *page_get_physical(uintptr_t seed); extern int linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp); extern int dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp); diff --git a/usr/src/uts/i86pc/vm/htable.c b/usr/src/uts/i86pc/vm/htable.c index e7ee3d5f45..c0eb1eed9d 100644 --- a/usr/src/uts/i86pc/vm/htable.c +++ b/usr/src/uts/i86pc/vm/htable.c @@ -20,7 +20,7 @@ */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -264,7 +264,7 @@ xen_map(uint64_t pte, caddr_t va) * A wrapper around page_get_physical(), with some extra checks. */ static pfn_t -ptable_alloc(void) +ptable_alloc(uintptr_t seed) { pfn_t pfn; page_t *pp; @@ -298,7 +298,7 @@ ptable_alloc(void) } #endif /* DEBUG */ - pp = page_get_physical(KM_NOSLEEP); + pp = page_get_physical(seed); if (pp == NULL) return (PFN_INVALID); ASSERT(PAGE_SHARED(pp)); @@ -326,13 +326,28 @@ ptable_free(pfn_t pfn) atomic_add_32(&active_ptables, -1); if (pp == NULL) panic("ptable_free(): no page for pfn!"); + ASSERT(PAGE_SHARED(pp)); ASSERT(pfn == pp->p_pagenum); ASSERT(!IN_XPV_PANIC()); + + /* + * Get an exclusive lock, might have to wait for a kmem reader. + */ + if (!page_tryupgrade(pp)) { + page_unlock(pp); + /* + * RFE: we could change this to not loop forever + * For now looping works - it's just like sfmmu. + */ + while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM)) + continue; + } #ifdef __xpv if (kpm_vbase && xen_kpm_page(pfn, PT_VALID | PT_WRITABLE) < 0) panic("failure making kpm r/w pfn=0x%lx", pfn); #endif - page_free_physical(pp); + page_free(pp, 1); + page_unresv(1); } /* @@ -767,7 +782,7 @@ htable_alloc( */ if (ht != NULL && !is_bare) { ht->ht_hat = hat; - ht->ht_pfn = ptable_alloc(); + ht->ht_pfn = ptable_alloc((uintptr_t)ht); if (ht->ht_pfn == PFN_INVALID) { if (USE_HAT_RESERVES()) htable_put_reserve(ht); @@ -830,7 +845,7 @@ htable_alloc( for (;;) { htable_t *stolen; - hat->hat_user_ptable = ptable_alloc(); + hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1); if (hat->hat_user_ptable != PFN_INVALID) break; stolen = htable_steal(1); diff --git a/usr/src/uts/i86pc/vm/vm_machdep.c b/usr/src/uts/i86pc/vm/vm_machdep.c index 11999bbe88..f83e760a8f 100644 --- a/usr/src/uts/i86pc/vm/vm_machdep.c +++ b/usr/src/uts/i86pc/vm/vm_machdep.c @@ -3724,25 +3724,34 @@ exec_get_spslew(void) * available - this would have a minimal impact on page coloring. */ page_t * -page_get_physical(int flags) +page_get_physical(uintptr_t seed) { page_t *pp; - u_offset_t offset = (u_offset_t)1 << 41; /* in VA hole */ + u_offset_t offset; static struct seg tmpseg; static uintptr_t ctr = 0; - static kmutex_t pgp_mutex; /* * This code is gross, we really need a simpler page allocator. * + * We need to assign an offset for the page to call page_create_va() * To avoid conflicts with other pages, we get creative with the offset. * For 32 bits, we need an offset > 4Gig * For 64 bits, need an offset somewhere in the VA hole. */ - if (page_resv(1, flags & KM_NOSLEEP) == 0) + offset = seed; + if (offset > kernelbase) + offset -= kernelbase; + offset <<= MMU_PAGESHIFT; +#if defined(__amd64) + offset += mmu.hole_start; /* something in VA hole */ +#else + offset += 1ULL << 40; /* something > 4 Gig */ +#endif + + if (page_resv(1, KM_NOSLEEP) == 0) return (NULL); - mutex_enter(&pgp_mutex); #ifdef DEBUG pp = page_exists(&kvp, offset); if (pp != NULL) @@ -3754,31 +3763,7 @@ page_get_physical(int flags) if (pp != NULL) { page_io_unlock(pp); page_hashout(pp, NULL); - mutex_exit(&pgp_mutex); page_downgrade(pp); - } else { - mutex_exit(&pgp_mutex); } return (pp); } - -void -page_free_physical(page_t *pp) -{ - /* - * Get an exclusive lock, might have to wait for a kmem reader. - */ - ASSERT(PAGE_SHARED(pp)); - if (!page_tryupgrade(pp)) { - page_unlock(pp); - /* - * RFE: we could change this to not loop forever - * George Cameron had some idea on how to do that. - * For now looping works - it's just like sfmmu. - */ - while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM)) - continue; - } - page_free(pp, 1); - page_unresv(1); -} |