|
|
|
|
@@ -143,7 +143,7 @@ struct gasket_coherent_page_entry {
|
|
|
|
|
u64 user_virt;
|
|
|
|
|
|
|
|
|
|
/* User virtual address that was mapped by the mmap kernel subsystem */
|
|
|
|
|
u64 kernel_virt;
|
|
|
|
|
dma_addr_t kernel_virt;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Whether this page has been mapped into a user land process virtual
|
|
|
|
|
@@ -528,7 +528,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* Does not perform validity checking.
|
|
|
|
|
*/
|
|
|
|
|
static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr)
|
|
|
|
|
u64 dev_addr)
|
|
|
|
|
{
|
|
|
|
|
return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) &
|
|
|
|
|
(pg_tbl->config.total_entries - 1);
|
|
|
|
|
@@ -539,7 +539,7 @@ static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* Does not perform validity checking.
|
|
|
|
|
*/
|
|
|
|
|
static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr)
|
|
|
|
|
u64 dev_addr)
|
|
|
|
|
{
|
|
|
|
|
return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) &
|
|
|
|
|
((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1);
|
|
|
|
|
@@ -550,7 +550,7 @@ static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* Does not perform validity checking.
|
|
|
|
|
*/
|
|
|
|
|
static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr)
|
|
|
|
|
u64 dev_addr)
|
|
|
|
|
{
|
|
|
|
|
return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) &
|
|
|
|
|
(GASKET_PAGES_PER_SUBTABLE - 1);
|
|
|
|
|
@@ -561,7 +561,7 @@ static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* The page table mutex must be held by the caller.
|
|
|
|
|
*/
|
|
|
|
|
static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_pages)
|
|
|
|
|
u64 dev_addr, uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
if (!gasket_is_pte_range_free(pg_tbl->entries +
|
|
|
|
|
gasket_simple_page_idx(pg_tbl, dev_addr),
|
|
|
|
|
@@ -629,7 +629,7 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* The page table mutex must be held by the caller.
|
|
|
|
|
*/
|
|
|
|
|
static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_pages)
|
|
|
|
|
u64 dev_addr, uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
uint slot = gasket_simple_page_idx(pg_tbl, dev_addr);
|
|
|
|
|
|
|
|
|
|
@@ -642,7 +642,7 @@ static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* The page table mutex must be held by the caller.
|
|
|
|
|
*/
|
|
|
|
|
static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_pages)
|
|
|
|
|
u64 dev_addr, uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
uint slot_idx, remain, len;
|
|
|
|
|
struct gasket_page_table_entry *pte;
|
|
|
|
|
@@ -673,7 +673,7 @@ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl,
|
|
|
|
|
|
|
|
|
|
/* Evaluates to nonzero if the specified virtual address is simple. */
|
|
|
|
|
static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong addr)
|
|
|
|
|
u64 addr)
|
|
|
|
|
{
|
|
|
|
|
return !((addr) & (pg_tbl)->extended_flag);
|
|
|
|
|
}
|
|
|
|
|
@@ -724,7 +724,7 @@ static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* currently-partitioned simple pages.
|
|
|
|
|
*/
|
|
|
|
|
static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_pages)
|
|
|
|
|
u64 dev_addr, uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
ulong page_offset = dev_addr & (PAGE_SIZE - 1);
|
|
|
|
|
ulong page_index =
|
|
|
|
|
@@ -732,7 +732,7 @@ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl,
|
|
|
|
|
|
|
|
|
|
if (gasket_components_to_dev_address(pg_tbl, 1, page_index,
|
|
|
|
|
page_offset) != dev_addr) {
|
|
|
|
|
dev_err(pg_tbl->device, "address is invalid, 0x%lX\n",
|
|
|
|
|
dev_err(pg_tbl->device, "address is invalid, 0x%llX\n",
|
|
|
|
|
dev_addr);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
@@ -762,18 +762,18 @@ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* currently-partitioned extended pages.
|
|
|
|
|
*/
|
|
|
|
|
static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_pages)
|
|
|
|
|
u64 dev_addr, uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
/* Starting byte index of dev_addr into the first mapped page */
|
|
|
|
|
ulong page_offset = dev_addr & (PAGE_SIZE - 1);
|
|
|
|
|
ulong page_global_idx, page_lvl0_idx;
|
|
|
|
|
ulong num_lvl0_pages;
|
|
|
|
|
ulong addr;
|
|
|
|
|
u64 addr;
|
|
|
|
|
|
|
|
|
|
/* check if the device address is out of bound */
|
|
|
|
|
addr = dev_addr & ~((pg_tbl)->extended_flag);
|
|
|
|
|
if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) {
|
|
|
|
|
dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n",
|
|
|
|
|
dev_err(pg_tbl->device, "device address out of bounds: 0x%llx\n",
|
|
|
|
|
dev_addr);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
@@ -791,7 +791,7 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl,
|
|
|
|
|
|
|
|
|
|
if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx,
|
|
|
|
|
page_offset) != dev_addr) {
|
|
|
|
|
dev_err(pg_tbl->device, "address is invalid: 0x%lx\n",
|
|
|
|
|
dev_err(pg_tbl->device, "address is invalid: 0x%llx\n",
|
|
|
|
|
dev_addr);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
@@ -819,7 +819,7 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* The page table mutex must be held by the caller.
|
|
|
|
|
*/
|
|
|
|
|
static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_pages)
|
|
|
|
|
u64 dev_addr, uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
if (!num_pages)
|
|
|
|
|
return;
|
|
|
|
|
@@ -844,7 +844,7 @@ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages);
|
|
|
|
|
if (ret) {
|
|
|
|
|
dev_err(pg_tbl->device,
|
|
|
|
|
"page table slots %u (@ 0x%lx) to %u are not available\n",
|
|
|
|
|
"page table slots %u (@ 0x%llx) to %u are not available\n",
|
|
|
|
|
slot_idx, dev_addr, slot_idx + num_pages - 1);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
@@ -920,7 +920,7 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl,
|
|
|
|
|
* The page table mutex must be held by the caller.
|
|
|
|
|
*/
|
|
|
|
|
static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl,
|
|
|
|
|
ulong dev_addr, uint num_entries)
|
|
|
|
|
u64 dev_addr, uint num_entries)
|
|
|
|
|
{
|
|
|
|
|
int ret = 0;
|
|
|
|
|
uint remain, subtable_slot_idx, len;
|
|
|
|
|
@@ -978,7 +978,7 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl,
|
|
|
|
|
if (ret) {
|
|
|
|
|
dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1;
|
|
|
|
|
dev_err(pg_tbl->device,
|
|
|
|
|
"page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are "
|
|
|
|
|
"page table slots (%lu,%llu) (@ 0x%lx) to (%lu,%lu) are "
|
|
|
|
|
"not available\n",
|
|
|
|
|
gasket_extended_lvl0_page_idx(pg_tbl, dev_addr),
|
|
|
|
|
dev_addr,
|
|
|
|
|
@@ -1061,7 +1061,7 @@ EXPORT_SYMBOL(gasket_page_table_map);
|
|
|
|
|
*
|
|
|
|
|
* The page table mutex is held for the entire operation.
|
|
|
|
|
*/
|
|
|
|
|
void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr,
|
|
|
|
|
void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, u64 dev_addr,
|
|
|
|
|
uint num_pages)
|
|
|
|
|
{
|
|
|
|
|
if (!num_pages)
|
|
|
|
|
@@ -1106,7 +1106,7 @@ void gasket_page_table_reset(struct gasket_page_table *pg_tbl)
|
|
|
|
|
|
|
|
|
|
/* See gasket_page_table.h for description. */
|
|
|
|
|
int gasket_page_table_lookup_page(
|
|
|
|
|
struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage,
|
|
|
|
|
struct gasket_page_table *pg_tbl, u64 dev_addr, struct page **ppage,
|
|
|
|
|
ulong *poffset)
|
|
|
|
|
{
|
|
|
|
|
uint page_num;
|
|
|
|
|
@@ -1168,7 +1168,7 @@ EXPORT_SYMBOL(gasket_page_table_are_addrs_bad);
|
|
|
|
|
|
|
|
|
|
/* See gasket_page_table.h for description. */
|
|
|
|
|
bool gasket_page_table_is_dev_addr_bad(
|
|
|
|
|
struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes)
|
|
|
|
|
struct gasket_page_table *pg_tbl, u64 dev_addr, ulong bytes)
|
|
|
|
|
{
|
|
|
|
|
uint num_pages = bytes / PAGE_SIZE;
|
|
|
|
|
|
|
|
|
|
@@ -1311,7 +1311,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
|
|
|
|
|
gasket_dev->page_table[index]->coherent_pages[j].paddr =
|
|
|
|
|
handle + j * PAGE_SIZE;
|
|
|
|
|
gasket_dev->page_table[index]->coherent_pages[j].kernel_virt =
|
|
|
|
|
(u64)mem + j * PAGE_SIZE;
|
|
|
|
|
(dma_addr_t)mem + j * PAGE_SIZE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (*dma_address == 0)
|
|
|
|
|
|