From 0305c1f77f4ecc613e8ca3a9241b4c856b96e945 Mon Sep 17 00:00:00 2001 From: Simon Que Date: Fri, 29 Jun 2018 22:49:38 -0400 Subject: [PATCH 001/209] drivers/staging: Gasket driver framework + Apex driver The Gasket (Google ASIC Software, Kernel Extensions, and Tools) kernel framework is a generic, flexible system that supports thin kernel drivers. Gasket kernel drivers are expected to handle opening and closing devices, mmap'ing BAR space as requested, a small selection of ioctls, and handling page table translation (covered below). Any other functions should be handled by userspace code. The Gasket common module is not enough to run a device. In order to customize the Gasket code for a given piece of hardware, a device specific module must be created. At a minimum, this module must define a struct gasket_driver_desc containing the device-specific data for use by the framework; in addition, the module must declare an __init function that calls gasket_register_device with the module's gasket_driver_desc struct. Finally, the driver must define an exit function that calls gasket_unregister_device with the module's gasket_driver_desc struct. One of the core assumptions of the Gasket framework is that precisely one process is allowed to have an open write handle to the device node at any given time. (That process may, once it has one write handle, open any number of additional write handles.) This is accomplished by tracking open and close data for each driver instance. Change-Id: I0ce1dc1dc9c533026adbc3aacaefb830ecdbc3e9 Signed-off-by: Rob Springer Signed-off-by: John Joseph Signed-off-by: Simon Que Signed-off-by: Greg Kroah-Hartman --- Kconfig | 23 + Makefile | 9 + TODO | 17 + apex.h | 95 ++ apex_driver.c | 771 ++++++++++++++++ gasket.h | 129 +++ gasket_constants.h | 56 ++ gasket_core.c | 2157 +++++++++++++++++++++++++++++++++++++++++++ gasket_core.h | 719 +++++++++++++++ gasket_interrupt.c | 638 +++++++++++++ gasket_interrupt.h | 172 ++++ gasket_ioctl.c | 449 +++++++++ gasket_ioctl.h | 35 + gasket_logging.h | 71 ++ gasket_page_table.c | 1771 +++++++++++++++++++++++++++++++++++ gasket_page_table.h | 265 ++++++ gasket_sysfs.c | 497 ++++++++++ gasket_sysfs.h | 199 ++++ 18 files changed, 8073 insertions(+) create mode 100644 Kconfig create mode 100644 Makefile create mode 100644 TODO create mode 100644 apex.h create mode 100644 apex_driver.c create mode 100644 gasket.h create mode 100644 gasket_constants.h create mode 100644 gasket_core.c create mode 100644 gasket_core.h create mode 100644 gasket_interrupt.c create mode 100644 gasket_interrupt.h create mode 100644 gasket_ioctl.c create mode 100644 gasket_ioctl.h create mode 100644 gasket_logging.h create mode 100644 gasket_page_table.c create mode 100644 gasket_page_table.h create mode 100644 gasket_sysfs.c create mode 100644 gasket_sysfs.h diff --git a/Kconfig b/Kconfig new file mode 100644 index 0000000..c836389 --- /dev/null +++ b/Kconfig @@ -0,0 +1,23 @@ +menu "Gasket devices" + +config STAGING_GASKET_FRAMEWORK + tristate "Gasket framework" + depends on PCI && X86_64 + help + This framework supports Gasket-compatible devices, such as Apex. + It is required for any of the following module(s). + + To compile this driver as a module, choose M here. The module + will be called "gasket". + +config STAGING_APEX_DRIVER + tristate "Apex Driver" + depends on STAGING_GASKET_FRAMEWORK + help + This driver supports the Apex device. Say Y if you want to + include this driver in the kernel. + + To compile this driver as a module, choose M here. The module + will be called "apex". + +endmenu diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cec813e --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +# +# Makefile for Gasket framework and dependent drivers. +# + +obj-$(CONFIG_STAGING_GASKET_FRAMEWORK) += gasket.o +obj-$(CONFIG_STAGING_APEX_DRIVER) += apex.o + +gasket-objs := gasket_core.o gasket_ioctl.o gasket_interrupt.o gasket_page_table.o gasket_sysfs.o +apex-objs := apex_driver.o diff --git a/TODO b/TODO new file mode 100644 index 0000000..0d8ee96 --- /dev/null +++ b/TODO @@ -0,0 +1,17 @@ +This is a list of things that need to be done to get this driver out of the +staging directory. +- Use SPDX tags to show the license of the file, and no more "boiler-plate" + license text is needed. +- Remove static function declarations. +- Document sysfs files with Documentation/ABI/ entries. +- Use misc interface instead of major number for driver version description. +- Add descriptions of module_param's +- Remove gasket-specific logging functions. +- apex_get_status() should actually check status. +- Static functions don't need kernel doc formatting, can be simplified. +- Fix multi-line alignment formatting to look like: + int ret = long_function_name(device, VARIABLE1, VARIABLE2, + VARIABLE3, VARIABLE4); +- "drivers" should never be dealing with "raw" sysfs calls or mess around with + kobjects at all. The driver core should handle all of this for you + automaically. There should not be a need for raw attribute macros. diff --git a/apex.h b/apex.h new file mode 100644 index 0000000..f2600aa --- /dev/null +++ b/apex.h @@ -0,0 +1,95 @@ +/* + * Apex kernel-userspace interface definition(s). + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __APEX_H__ +#define __APEX_H__ + +#include +#include + +#include "gasket.h" + +/* Structural definitions/macros. */ +/* The number of PCI BARs. */ +#define APEX_NUM_BARS 3 + +/* Size of a memory page in bytes, and the related number of bits to shift. */ +#define APEX_PAGE_SHIFT 12 +#define APEX_PAGE_SIZE BIT(APEX_PAGE_SHIFT) + +#define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */ + +/* Addresses are 2^3=8 bytes each. */ +/* page in second level page table */ +/* holds APEX_PAGE_SIZE/8 addresses */ +#define APEX_ADDR_SHIFT 3 +#define APEX_LEVEL_SHIFT (APEX_PAGE_SHIFT - APEX_ADDR_SHIFT) +#define APEX_LEVEL_SIZE BIT(APEX_LEVEL_SHIFT) + +#define APEX_PAGE_TABLE_MAX 65536 +#define APEX_SIMPLE_PAGE_MAX APEX_PAGE_TABLE_MAX +#define APEX_EXTENDED_PAGE_MAX (APEX_PAGE_TABLE_MAX << APEX_LEVEL_SHIFT) + +/* Check reset 120 times */ +#define APEX_RESET_RETRY 120 +/* Wait 100 ms between checks. Total 12 sec wait maximum. */ +#define APEX_RESET_DELAY 100 + +#define APEX_CHIP_INIT_DONE 2 +#define APEX_RESET_ACCEPTED 0 + +enum apex_reset_types { + APEX_CHIP_REINIT_RESET = 3, +}; + +/* Interrupt defines */ +/* Gasket device interrupts enums must be dense (i.e., no empty slots). */ +enum apex_interrupt { + APEX_INTERRUPT_INSTR_QUEUE = 0, + APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1, + APEX_INTERRUPT_PARAM_QUEUE = 2, + APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3, + APEX_INTERRUPT_SC_HOST_0 = 4, + APEX_INTERRUPT_SC_HOST_1 = 5, + APEX_INTERRUPT_SC_HOST_2 = 6, + APEX_INTERRUPT_SC_HOST_3 = 7, + APEX_INTERRUPT_TOP_LEVEL_0 = 8, + APEX_INTERRUPT_TOP_LEVEL_1 = 9, + APEX_INTERRUPT_TOP_LEVEL_2 = 10, + APEX_INTERRUPT_TOP_LEVEL_3 = 11, + APEX_INTERRUPT_FATAL_ERR = 12, + APEX_INTERRUPT_COUNT = 13, +}; + +/* + * Clock Gating ioctl. + */ +struct apex_gate_clock_ioctl { + /* Enter or leave clock gated state. */ + u64 enable; + + /* If set, enter clock gating state, regardless of custom block's + * internal idle state + */ + u64 force_idle; +}; + +/* Base number for all Apex-common IOCTLs */ +#define APEX_IOCTL_BASE 0x7F + +/* Enable/Disable clock gating. */ +#define APEX_IOCTL_GATE_CLOCK \ + _IOW(APEX_IOCTL_BASE, 0, struct apex_gate_clock_ioctl) + +#endif /* __APEX_H__ */ diff --git a/apex_driver.c b/apex_driver.c new file mode 100644 index 0000000..3952567 --- /dev/null +++ b/apex_driver.c @@ -0,0 +1,771 @@ +/* Driver for the Apex chip. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apex.h" + +#include "gasket_core.h" +#include "gasket_interrupt.h" +#include "gasket_logging.h" +#include "gasket_page_table.h" +#include "gasket_sysfs.h" + +/* Constants */ +#define APEX_DEVICE_NAME "Apex" +#define APEX_DRIVER_VERSION "1.0" + +/* CSRs are in BAR 2. */ +#define APEX_BAR_INDEX 2 + +#define APEX_PCI_VENDOR_ID 0x1ac1 +#define APEX_PCI_DEVICE_ID 0x089a + +/* Bar Offsets. */ +#define APEX_BAR_OFFSET 0 +#define APEX_CM_OFFSET 0x1000000 + +/* The sizes of each Apex BAR 2. */ +#define APEX_BAR_BYTES 0x100000 +#define APEX_CH_MEM_BYTES (PAGE_SIZE * MAX_NUM_COHERENT_PAGES) + +/* The number of user-mappable memory ranges in BAR2 of a Apex chip. */ +#define NUM_REGIONS 3 + +/* The number of nodes in a Apex chip. */ +#define NUM_NODES 1 + +/* + * The total number of entries in the page table. Should match the value read + * from the register APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE. + */ +#define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192 + +/* Enumeration of the supported sysfs entries. */ +enum sysfs_attribute_type { + ATTR_KERNEL_HIB_PAGE_TABLE_SIZE, + ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE, + ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES, +}; + +/* + * Register offsets into BAR2 memory. + * Only values necessary for driver implementation are defined. + */ +enum apex_bar2_regs { + APEX_BAR2_REG_SCU_BASE = 0x1A300, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE = 0x46000, + APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE = 0x46008, + APEX_BAR2_REG_KERNEL_HIB_TRANSLATION_ENABLE = 0x46010, + APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL = 0x46018, + APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL = 0x46020, + APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL = 0x46028, + APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL = 0x46030, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL = 0x46038, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL = 0x46040, + APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL = 0x46048, + APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE = 0x46050, + APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE_MASK = 0x46058, + APEX_BAR2_REG_KERNEL_HIB_STATUS_BLOCK_DELAY = 0x46060, + APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY0 = 0x46068, + APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY1 = 0x46070, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT = 0x46078, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT = 0x46080, + APEX_BAR2_REG_KERNEL_WIRE_INT_PENDING_BIT_ARRAY = 0x48778, + APEX_BAR2_REG_KERNEL_WIRE_INT_MASK_ARRAY = 0x48780, + APEX_BAR2_REG_USER_HIB_DMA_PAUSE = 0x486D8, + APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0, + APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000, + + /* Error registers - Used mostly for debug */ + APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0, + APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS = 0x41a0, +}; + +/* Addresses for packed registers. */ +#define APEX_BAR2_REG_AXI_QUIESCE (APEX_BAR2_REG_SCU_BASE + 0x2C) +#define APEX_BAR2_REG_GCB_CLOCK_GATE (APEX_BAR2_REG_SCU_BASE + 0x14) +#define APEX_BAR2_REG_SCU_0 (APEX_BAR2_REG_SCU_BASE + 0xc) +#define APEX_BAR2_REG_SCU_1 (APEX_BAR2_REG_SCU_BASE + 0x10) +#define APEX_BAR2_REG_SCU_2 (APEX_BAR2_REG_SCU_BASE + 0x14) +#define APEX_BAR2_REG_SCU_3 (APEX_BAR2_REG_SCU_BASE + 0x18) +#define APEX_BAR2_REG_SCU_4 (APEX_BAR2_REG_SCU_BASE + 0x1c) +#define APEX_BAR2_REG_SCU_5 (APEX_BAR2_REG_SCU_BASE + 0x20) + +#define SCU3_RG_PWR_STATE_OVR_BIT_OFFSET 26 +#define SCU3_RG_PWR_STATE_OVR_MASK_WIDTH 2 +#define SCU3_CUR_RST_GCB_BIT_MASK 0x10 +#define SCU2_RG_RST_GCB_BIT_MASK 0xc + +/* Configuration for page table. */ +static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = { + { + .id = 0, + .mode = GASKET_PAGE_TABLE_MODE_NORMAL, + .total_entries = APEX_PAGE_TABLE_TOTAL_ENTRIES, + .base_reg = APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE, + .extended_reg = APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE, + .extended_bit = APEX_EXTENDED_SHIFT, + }, +}; + +/* Function declarations */ +static int __init apex_init(void); +static void apex_exit(void); + +static int apex_add_dev_cb(struct gasket_dev *gasket_dev); + +static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev); + +static int apex_device_cleanup(struct gasket_dev *gasket_dev); + +static int apex_device_open_cb(struct gasket_dev *gasket_dev); + +static ssize_t sysfs_show( + struct device *device, struct device_attribute *attr, char *buf); + +static int apex_reset(struct gasket_dev *gasket_dev, uint type); + +static int apex_get_status(struct gasket_dev *gasket_dev); + +static uint apex_ioctl_check_permissions(struct file *file, uint cmd); + +static long apex_ioctl(struct file *file, uint cmd, ulong arg); + +static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg); + +static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type); + +static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type); + +static bool is_gcb_in_reset(struct gasket_dev *gasket_dev); + +/* Data definitions */ + +/* The data necessary to display this file's sysfs entries. */ +static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { + GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show, + ATTR_KERNEL_HIB_PAGE_TABLE_SIZE), + GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show, + ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE), + GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, + ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), + GASKET_END_OF_ATTR_ARRAY +}; + +static const struct pci_device_id apex_pci_ids[] = { + { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 } +}; + +/* The regions in the BAR2 space that can be mapped into user space. */ +static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = { + { 0x40000, 0x1000 }, + { 0x44000, 0x1000 }, + { 0x48000, 0x1000 }, +}; + +static const struct gasket_mappable_region cm_mappable_regions[1] = { { 0x0, + APEX_CH_MEM_BYTES } }; + +/* Interrupt descriptors for Apex */ +static struct gasket_interrupt_desc apex_interrupts[] = { + { + APEX_INTERRUPT_INSTR_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL, + UNPACKED, + }, + { + APEX_INTERRUPT_INPUT_ACTV_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL, + UNPACKED + }, + { + APEX_INTERRUPT_PARAM_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL, + UNPACKED + }, + { + APEX_INTERRUPT_OUTPUT_ACTV_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL, + UNPACKED + }, + { + APEX_INTERRUPT_SC_HOST_0, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_0 + }, + { + APEX_INTERRUPT_SC_HOST_1, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_1 + }, + { + APEX_INTERRUPT_SC_HOST_2, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_2 + }, + { + APEX_INTERRUPT_SC_HOST_3, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_3 + }, + { + APEX_INTERRUPT_TOP_LEVEL_0, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_0 + }, + { + APEX_INTERRUPT_TOP_LEVEL_1, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_1 + }, + { + APEX_INTERRUPT_TOP_LEVEL_2, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_2 + }, + { + APEX_INTERRUPT_TOP_LEVEL_3, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_3 + }, + { + APEX_INTERRUPT_FATAL_ERR, + APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL, + UNPACKED + }, +}; + +static struct gasket_driver_desc apex_desc = { + .name = "apex", + .driver_version = APEX_DRIVER_VERSION, + .major = 120, + .minor = 0, + .module = THIS_MODULE, + .pci_id_table = apex_pci_ids, + + .num_page_tables = NUM_NODES, + .page_table_bar_index = APEX_BAR_INDEX, + .page_table_configs = apex_page_table_configs, + .page_table_extended_bit = APEX_EXTENDED_SHIFT, + + .bar_descriptions = { + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET, + NUM_REGIONS, mappable_regions, PCI_BAR }, + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + }, + .coherent_buffer_description = { + APEX_CH_MEM_BYTES, + (VM_WRITE | VM_READ), + APEX_CM_OFFSET, + }, + .interrupt_type = PCI_MSIX, + .interrupt_bar_index = APEX_BAR_INDEX, + .num_interrupts = APEX_INTERRUPT_COUNT, + .interrupts = apex_interrupts, + .interrupt_pack_width = 7, + + .add_dev_cb = apex_add_dev_cb, + .remove_dev_cb = NULL, + + .enable_dev_cb = NULL, + .disable_dev_cb = NULL, + + .sysfs_setup_cb = apex_sysfs_setup_cb, + .sysfs_cleanup_cb = NULL, + + .device_open_cb = apex_device_open_cb, + .device_close_cb = apex_device_cleanup, + + .ioctl_handler_cb = apex_ioctl, + .device_status_cb = apex_get_status, + .hardware_revision_cb = NULL, + .device_reset_cb = apex_reset, +}; + +/* Module registration boilerplate */ +MODULE_DESCRIPTION("Google Apex driver"); +MODULE_VERSION(APEX_DRIVER_VERSION); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("John Joseph "); +MODULE_DEVICE_TABLE(pci, apex_pci_ids); +module_init(apex_init); +module_exit(apex_exit); + +/* Allows device to enter power save upon driver close(). */ +static int allow_power_save; + +/* Allows SW based clock gating. */ +static int allow_sw_clock_gating; + +/* Allows HW based clock gating. */ +/* Note: this is not mutual exclusive with SW clock gating. */ +static int allow_hw_clock_gating = 1; + +/* Act as if only GCB is instantiated. */ +static int bypass_top_level; + +module_param(allow_power_save, int, 0644); +module_param(allow_sw_clock_gating, int, 0644); +module_param(allow_hw_clock_gating, int, 0644); +module_param(bypass_top_level, int, 0644); + +static int __init apex_init(void) +{ + return gasket_register_device(&apex_desc); +} + +static void apex_exit(void) +{ + gasket_unregister_device(&apex_desc); +} + +static int apex_add_dev_cb(struct gasket_dev *gasket_dev) +{ + ulong page_table_ready, msix_table_ready; + int retries = 0; + + gasket_log_error(gasket_dev, "apex_add_dev_cb."); + + apex_reset(gasket_dev, 0); + + while (retries < APEX_RESET_RETRY) { + page_table_ready = + gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); + msix_table_ready = + gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); + if (page_table_ready && msix_table_ready) + break; + schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); + retries++; + } + + if (retries == APEX_RESET_RETRY) { + if (!page_table_ready) + gasket_log_error( + gasket_dev, "Page table init timed out."); + if (!msix_table_ready) + gasket_log_error( + gasket_dev, "MSI-X table init timed out."); + return -ETIMEDOUT; + } + + return 0; +} + +static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) +{ + return gasket_sysfs_create_entries( + gasket_dev->dev_info.device, apex_sysfs_attrs); +} + +/* On device open, we want to perform a core reinit reset. */ +static int apex_device_open_cb(struct gasket_dev *gasket_dev) +{ + return gasket_reset_nolock(gasket_dev, APEX_CHIP_REINIT_RESET); +} + +/** + * apex_get_status - Set device status. + * @dev: Apex device struct. + * + * Description: Check the device status registers and set the driver status + * to ALIVE or DEAD. + * + * Returns 0 if status is ALIVE, a negative error number otherwise. + */ +static int apex_get_status(struct gasket_dev *gasket_dev) +{ + /* TODO: Check device status. */ + return GASKET_STATUS_ALIVE; +} + +/** + * apex_device_cleanup - Clean up Apex HW after close. + * @gasket_dev: Gasket device pointer. + * + * Description: Resets the Apex hardware. Called on final close via + * device_close_cb. + */ +static int apex_device_cleanup(struct gasket_dev *gasket_dev) +{ + u64 scalar_error; + u64 hib_error; + int ret = 0; + + hib_error = gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_ERROR_STATUS); + scalar_error = gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); + + gasket_log_info( + gasket_dev, + "apex_device_cleanup 0x%p hib_error 0x%llx scalar_error " + "0x%llx.", + gasket_dev, hib_error, scalar_error); + + if (allow_power_save) + ret = apex_enter_reset(gasket_dev, APEX_CHIP_REINIT_RESET); + + return ret; +} + +/** + * apex_reset - Quits reset. + * @gasket_dev: Gasket device pointer. + * + * Description: Resets the hardware, then quits reset. + * Called on device open. + * + */ +static int apex_reset(struct gasket_dev *gasket_dev, uint type) +{ + int ret; + + if (bypass_top_level) + return 0; + + gasket_log_debug(gasket_dev, "apex_reset."); + + if (!is_gcb_in_reset(gasket_dev)) { + /* We are not in reset - toggle the reset bit so as to force + * re-init of custom block + */ + gasket_log_debug(gasket_dev, "apex_reset: toggle reset."); + + ret = apex_enter_reset(gasket_dev, type); + if (ret) + return ret; + } + ret = apex_quit_reset(gasket_dev, type); + + return ret; +} + +/* + * Enters GCB reset state. + */ +static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) +{ + if (bypass_top_level) + return 0; + + gasket_log_debug(gasket_dev, "apex_enter_reset."); + + /* + * Software reset: + * Enable sleep mode + * - Software force GCB idle + * - Enable GCB idle + */ + gasket_read_modify_write_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER, 0x0, 1, 32); + + /* - Initiate DMA pause */ + gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_DMA_PAUSE); + + /* - Wait for DMA pause complete. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + gasket_log_error(gasket_dev, + "DMAs did not quiece within timeout (%d ms)", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -EINVAL; + } + + /* - Enable GCB reset (0x1 to rg_rst_gcb) */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x1, 2, 2); + + /* - Enable GCB clock Gate (0x1 to rg_gated_gcb) */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x1, 2, 18); + + /* - Enable GCB memory shut down (0x3 to rg_force_ram_sd) */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x3, 2, 14); + + /* - Wait for RAM shutdown. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 1 << 6, 1 << 6, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + gasket_log_error( + gasket_dev, + "RAM did not shut down within timeout (%d ms)", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -EINVAL; + } + + return 0; +} + +/* + * Quits GCB reset state. + */ +static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) +{ + u32 val0, val1; + + if (bypass_top_level) + return 0; + + gasket_log_debug(gasket_dev, "apex_quit_reset."); + + /* + * Disable sleep mode: + * - Disable GCB memory shut down: + * - b00: Not forced (HW controlled) + * - b1x: Force disable + */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x0, 2, 14); + + /* + * - Disable software clock gate: + * - b00: Not forced (HW controlled) + * - b1x: Force disable + */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x0, 2, 18); + + /* + * - Disable GCB reset (rg_rst_gcb): + * - b00: Not forced (HW controlled) + * - b1x: Force disable = Force not Reset + */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x2, 2, 2); + + /* - Wait for RAM enable. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 1 << 6, 0, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + gasket_log_error( + gasket_dev, + "RAM did not enable within timeout (%d ms)", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -EINVAL; + } + + /* - Wait for Reset complete. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, + SCU3_CUR_RST_GCB_BIT_MASK, 0, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + gasket_log_error( + gasket_dev, + "GCB did not leave reset within timeout (%d ms)", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -EINVAL; + } + + if (!allow_hw_clock_gating) { + val0 = gasket_dev_read_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + /* Inactive and Sleep mode are disabled. */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x3, + SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, + SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); + val1 = gasket_dev_read_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + gasket_log_error( + gasket_dev, "Disallow HW clock gating 0x%x -> 0x%x", + val0, val1); + } else { + val0 = gasket_dev_read_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + /* Inactive mode enabled - Sleep mode disabled. */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 2, + SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, + SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); + val1 = gasket_dev_read_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + gasket_log_error( + gasket_dev, "Allow HW clock gating 0x%x -> 0x%x", val0, + val1); + } + + return 0; +} + +/* + * Determines if GCB is in reset state. + */ +static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) +{ + u32 val = gasket_dev_read_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + + /* Masks rg_rst_gcb bit of SCU_CTRL_2 */ + return (val & SCU3_CUR_RST_GCB_BIT_MASK); +} + +/* + * Check permissions for Apex ioctls. + * @file: File pointer from ioctl. + * @cmd: ioctl command. + * + * Returns 1 if the current user may execute this ioctl, and 0 otherwise. + */ +static uint apex_ioctl_check_permissions(struct file *filp, uint cmd) +{ + struct gasket_dev *gasket_dev = filp->private_data; + int root = capable(CAP_SYS_ADMIN); + int is_owner = gasket_dev->dev_info.ownership.is_owned && + current->tgid == gasket_dev->dev_info.ownership.owner; + + if (root || is_owner) + return 1; + return 0; +} + +/* + * Apex-specific ioctl handler. + */ +static long apex_ioctl(struct file *filp, uint cmd, ulong arg) +{ + struct gasket_dev *gasket_dev = filp->private_data; + + if (!apex_ioctl_check_permissions(filp, cmd)) + return -EPERM; + + switch (cmd) { + case APEX_IOCTL_GATE_CLOCK: + return apex_clock_gating(gasket_dev, arg); + default: + return -ENOTTY; /* unknown command */ + } +} + +/* + * Gates or un-gates Apex clock. + * @gasket_dev: Gasket device pointer. + * @arg: User ioctl arg, in this case to a apex_gate_clock_ioctl struct. + */ +static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg) +{ + struct apex_gate_clock_ioctl ibuf; + + if (bypass_top_level) + return 0; + + if (allow_sw_clock_gating) { + if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf))) + return -EFAULT; + + gasket_log_error( + gasket_dev, "apex_clock_gating %llu", ibuf.enable); + + if (ibuf.enable) { + /* Quiesce AXI, gate GCB clock. */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, 16); + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, 2, 18); + } else { + /* Un-gate GCB clock, un-quiesce AXI. */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, 2, 18); + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, 16); + } + } + return 0; +} + +/* + * Display driver sysfs entries. + * @device: Kernel device structure. + * @attr: Attribute to display. + * @buf: Buffer to which to write output. + * + * Description: Looks up the driver data and file-specific attribute data (the + * type of the attribute), then fills "buf" accordingly. + */ +static ssize_t sysfs_show( + struct device *device, struct device_attribute *attr, char *buf) +{ + int ret; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + enum sysfs_attribute_type type; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + gasket_nodev_error("No Apex device sysfs mapping found"); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + gasket_nodev_error("No Apex device sysfs attr data found"); + gasket_sysfs_put_device_data(device, gasket_dev); + return 0; + } + + type = (enum sysfs_attribute_type)gasket_sysfs_get_attr(device, attr); + switch (type) { + case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE: + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + gasket_page_table_num_entries( + gasket_dev->page_table[0])); + break; + case ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE: + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + gasket_page_table_num_entries( + gasket_dev->page_table[0])); + break; + case ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES: + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + gasket_page_table_num_active_pages( + gasket_dev->page_table[0])); + break; + default: + gasket_log_error( + gasket_dev, "Unknown attribute: %s", attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} diff --git a/gasket.h b/gasket.h new file mode 100644 index 0000000..593d508 --- /dev/null +++ b/gasket.h @@ -0,0 +1,129 @@ +/* Common Gasket device kernel and user space declarations. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __GASKET_H__ +#define __GASKET_H__ + +#include +#include + +/* ioctl structure declarations */ + +/* Ioctl structures are padded to a multiple of 64 bits */ +/* and padded to put 64 bit values on 64 bit boundaries. */ +/* Unsigned 64 bit integers are used to hold pointers. */ +/* This helps compatibility between 32 and 64 bits. */ + +/* + * Common structure for ioctls associating an eventfd with a device interrupt, + * when using the Gasket interrupt module. + */ +struct gasket_interrupt_eventfd { + u64 interrupt; + u64 event_fd; +}; + +/* + * Common structure for ioctls mapping and unmapping buffers when using the + * Gasket page_table module. + */ +struct gasket_page_table_ioctl { + u64 page_table_index; + u64 size; + u64 host_address; + u64 device_address; +}; + +/* + * Common structure for ioctls mapping and unmapping buffers when using the + * Gasket page_table module. + * dma_address: phys addr start of coherent memory, allocated by kernel + */ +struct gasket_coherent_alloc_config_ioctl { + u64 page_table_index; + u64 enable; + u64 size; + u64 dma_address; +}; + +/* Base number for all Gasket-common IOCTLs */ +#define GASKET_IOCTL_BASE 0xDC + +/* Reset the device using the specified reset type. */ +#define GASKET_IOCTL_RESET _IOW(GASKET_IOCTL_BASE, 0, unsigned long) + +/* Associate the specified [event]fd with the specified interrupt. */ +#define GASKET_IOCTL_SET_EVENTFD \ + _IOW(GASKET_IOCTL_BASE, 1, struct gasket_interrupt_eventfd) + +/* + * Clears any eventfd associated with the specified interrupt. The (ulong) + * argument is the interrupt number to clear. + */ +#define GASKET_IOCTL_CLEAR_EVENTFD _IOW(GASKET_IOCTL_BASE, 2, unsigned long) + +/* + * [Loopbacks only] Requests that the loopback device send the specified + * interrupt to the host. The (ulong) argument is the number of the interrupt to + * send. + */ +#define GASKET_IOCTL_LOOPBACK_INTERRUPT \ + _IOW(GASKET_IOCTL_BASE, 3, unsigned long) + +/* Queries the kernel for the number of page tables supported by the device. */ +#define GASKET_IOCTL_NUMBER_PAGE_TABLES _IOR(GASKET_IOCTL_BASE, 4, u64) + +/* + * Queries the kernel for the maximum size of the page table. Only the size and + * page_table_index fields are used from the struct gasket_page_table_ioctl. + */ +#define GASKET_IOCTL_PAGE_TABLE_SIZE \ + _IOWR(GASKET_IOCTL_BASE, 5, struct gasket_page_table_ioctl) + +/* + * Queries the kernel for the current simple page table size. Only the size and + * page_table_index fields are used from the struct gasket_page_table_ioctl. + */ +#define GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE \ + _IOWR(GASKET_IOCTL_BASE, 6, struct gasket_page_table_ioctl) + +/* + * Tells the kernel to change the split between the number of simple and + * extended entries in the given page table. Only the size and page_table_index + * fields are used from the struct gasket_page_table_ioctl. + */ +#define GASKET_IOCTL_PARTITION_PAGE_TABLE \ + _IOW(GASKET_IOCTL_BASE, 7, struct gasket_page_table_ioctl) + +/* + * Tells the kernel to map size bytes at host_address to device_address in + * page_table_index page table. + */ +#define GASKET_IOCTL_MAP_BUFFER \ + _IOW(GASKET_IOCTL_BASE, 8, struct gasket_page_table_ioctl) + +/* + * Tells the kernel to unmap size bytes at host_address from device_address in + * page_table_index page table. + */ +#define GASKET_IOCTL_UNMAP_BUFFER \ + _IOW(GASKET_IOCTL_BASE, 9, struct gasket_page_table_ioctl) + +/* Clear the interrupt counts stored for this device. */ +#define GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS _IO(GASKET_IOCTL_BASE, 10) + +/* Enable/Disable and configure the coherent allocator. */ +#define GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR \ + _IOWR(GASKET_IOCTL_BASE, 11, struct gasket_coherent_alloc_config_ioctl) + +#endif /* __GASKET_H__ */ diff --git a/gasket_constants.h b/gasket_constants.h new file mode 100644 index 0000000..b39e3e3 --- /dev/null +++ b/gasket_constants.h @@ -0,0 +1,56 @@ +/* Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __GASKET_CONSTANTS_H__ +#define __GASKET_CONSTANTS_H__ + +#define GASKET_FRAMEWORK_VERSION "1.1.1" + +/* + * The maximum number of simultaneous device types supported by the framework. + */ +#define GASKET_FRAMEWORK_DESC_MAX 2 + +/* The maximum devices per each type. */ +#define GASKET_DEV_MAX 256 + +/* The number of supported (and possible) PCI BARs. */ +#define GASKET_NUM_BARS 6 + +/* The number of supported Gasket page tables per device. */ +#define GASKET_MAX_NUM_PAGE_TABLES 1 + +/* Maximum length of device names (driver name + minor number suffix + NULL). */ +#define GASKET_NAME_MAX 32 + +/* Device status enumeration. */ +enum gasket_status { + /* + * A device is DEAD if it has not been initialized or has had an error. + */ + GASKET_STATUS_DEAD = 0, + /* + * A device is LAMED if the hardware is healthy but the kernel was + * unable to enable some functionality (e.g. interrupts). + */ + GASKET_STATUS_LAMED, + + /* A device is ALIVE if it is ready for operation. */ + GASKET_STATUS_ALIVE, + + /* + * This status is set when the driver is exiting and waiting for all + * handles to be closed. + */ + GASKET_STATUS_DRIVER_EXIT, +}; + +#endif diff --git a/gasket_core.c b/gasket_core.c new file mode 100644 index 0000000..6511a33 --- /dev/null +++ b/gasket_core.c @@ -0,0 +1,2157 @@ +/* Gasket generic driver framework. This file contains the implementation + * for the Gasket generic driver framework - the functionality that is common + * across Gasket devices. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include "gasket_core.h" + +#include "gasket_interrupt.h" +#include "gasket_ioctl.h" +#include "gasket_logging.h" +#include "gasket_page_table.h" +#include "gasket_sysfs.h" + +#include +#include +#include + +#ifdef GASKET_KERNEL_TRACE_SUPPORT +#define CREATE_TRACE_POINTS +#include +#else +#define trace_gasket_mmap_exit(x) +#define trace_gasket_mmap_entry(x, ...) +#endif + +/* + * "Private" members of gasket_driver_desc. + * + * Contains internal per-device type tracking data, i.e., data not appropriate + * as part of the public interface for the generic framework. + */ +struct gasket_internal_desc { + /* Device-specific-driver-provided configuration information. */ + const struct gasket_driver_desc *driver_desc; + + /* Protects access to per-driver data (i.e. this structure). */ + struct mutex mutex; + + /* Kernel-internal device class. */ + struct class *class; + + /* PCI subsystem metadata associated with this driver. */ + struct pci_driver pci; + + /* Instantiated / present devices of this type. */ + struct gasket_dev *devs[GASKET_DEV_MAX]; +}; + +/* do_map_region() needs be able to return more than just true/false. */ +enum do_map_region_status { + /* The region was successfully mapped. */ + DO_MAP_REGION_SUCCESS, + + /* Attempted to map region and failed. */ + DO_MAP_REGION_FAILURE, + + /* The requested region to map was not part of a mappable region. */ + DO_MAP_REGION_INVALID, +}; + +/* Function declarations; comments are with definitions. */ +static int __init gasket_init(void); +static void __exit gasket_exit(void); + +static int gasket_pci_probe( + struct pci_dev *pci_dev, const struct pci_device_id *id); +static void gasket_pci_remove(struct pci_dev *pci_dev); + +static int gasket_setup_pci(struct pci_dev *pci_dev, struct gasket_dev *dev); +static void gasket_cleanup_pci(struct gasket_dev *dev); + +static int gasket_map_pci_bar(struct gasket_dev *dev, int bar_num); +static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num); + +static int gasket_alloc_dev( + struct gasket_internal_desc *internal_desc, struct device *dev, + struct gasket_dev **pdev, const char *kobj_name); +static void gasket_free_dev(struct gasket_dev *dev); + +static int gasket_find_dev_slot( + struct gasket_internal_desc *internal_desc, const char *kobj_name); + +static int gasket_add_cdev( + struct gasket_cdev_info *dev_info, + const struct file_operations *file_ops, struct module *owner); + +static int gasket_enable_dev( + struct gasket_internal_desc *internal_desc, + struct gasket_dev *gasket_dev); +static void gasket_disable_dev(struct gasket_dev *gasket_dev); + +static struct gasket_internal_desc *lookup_internal_desc( + struct pci_dev *pci_dev); + +static ssize_t gasket_sysfs_data_show( + struct device *device, struct device_attribute *attr, char *buf); + +static int gasket_mmap(struct file *filp, struct vm_area_struct *vma); +static int gasket_open(struct inode *inode, struct file *file); +static int gasket_release(struct inode *inode, struct file *file); +static long gasket_ioctl(struct file *filp, uint cmd, ulong arg); + +static int gasket_mm_vma_bar_offset( + const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, + ulong *bar_offset); +static bool gasket_mm_get_mapping_addrs( + const struct gasket_mappable_region *region, ulong bar_offset, + ulong requested_length, struct gasket_mappable_region *mappable_region, + ulong *virt_offset); +static enum do_map_region_status do_map_region( + const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, + struct gasket_mappable_region *map_region); + +static int gasket_get_hw_status(struct gasket_dev *gasket_dev); + +/* Global data definitions. */ +/* Mutex - only for framework-wide data. Other data should be protected by + * finer-grained locks. + */ +static DEFINE_MUTEX(g_mutex); + +/* List of all registered device descriptions & their supporting data. */ +static struct gasket_internal_desc g_descs[GASKET_FRAMEWORK_DESC_MAX]; + +/* Mapping of statuses to human-readable strings. Must end with {0,NULL}. */ +static const struct gasket_num_name gasket_status_name_table[] = { + { GASKET_STATUS_DEAD, "DEAD" }, + { GASKET_STATUS_ALIVE, "ALIVE" }, + { GASKET_STATUS_LAMED, "LAMED" }, + { GASKET_STATUS_DRIVER_EXIT, "DRIVER_EXITING" }, + { 0, NULL }, +}; + +/* Enumeration of the automatic Gasket framework sysfs nodes. */ +enum gasket_sysfs_attribute_type { + ATTR_BAR_OFFSETS, + ATTR_BAR_SIZES, + ATTR_DRIVER_VERSION, + ATTR_FRAMEWORK_VERSION, + ATTR_DEVICE_TYPE, + ATTR_HARDWARE_REVISION, + ATTR_PCI_ADDRESS, + ATTR_STATUS, + ATTR_IS_DEVICE_OWNED, + ATTR_DEVICE_OWNER, + ATTR_WRITE_OPEN_COUNT, + ATTR_RESET_COUNT, + ATTR_USER_MEM_RANGES +}; + +/* File operations for all Gasket devices. */ +static const struct file_operations gasket_file_ops = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .mmap = gasket_mmap, + .open = gasket_open, + .release = gasket_release, + .unlocked_ioctl = gasket_ioctl, +}; + +/* These attributes apply to all Gasket driver instances. */ +static const struct gasket_sysfs_attribute gasket_sysfs_generic_attrs[] = { + GASKET_SYSFS_RO(bar_offsets, gasket_sysfs_data_show, ATTR_BAR_OFFSETS), + GASKET_SYSFS_RO(bar_sizes, gasket_sysfs_data_show, ATTR_BAR_SIZES), + GASKET_SYSFS_RO(driver_version, gasket_sysfs_data_show, + ATTR_DRIVER_VERSION), + GASKET_SYSFS_RO(framework_version, gasket_sysfs_data_show, + ATTR_FRAMEWORK_VERSION), + GASKET_SYSFS_RO(device_type, gasket_sysfs_data_show, ATTR_DEVICE_TYPE), + GASKET_SYSFS_RO(revision, gasket_sysfs_data_show, + ATTR_HARDWARE_REVISION), + GASKET_SYSFS_RO(pci_address, gasket_sysfs_data_show, ATTR_PCI_ADDRESS), + GASKET_SYSFS_RO(status, gasket_sysfs_data_show, ATTR_STATUS), + GASKET_SYSFS_RO(is_device_owned, gasket_sysfs_data_show, + ATTR_IS_DEVICE_OWNED), + GASKET_SYSFS_RO(device_owner, gasket_sysfs_data_show, + ATTR_DEVICE_OWNER), + GASKET_SYSFS_RO(write_open_count, gasket_sysfs_data_show, + ATTR_WRITE_OPEN_COUNT), + GASKET_SYSFS_RO(reset_count, gasket_sysfs_data_show, ATTR_RESET_COUNT), + GASKET_SYSFS_RO(user_mem_ranges, gasket_sysfs_data_show, + ATTR_USER_MEM_RANGES), + GASKET_END_OF_ATTR_ARRAY +}; + +MODULE_DESCRIPTION("Google Gasket driver framework"); +MODULE_VERSION(GASKET_FRAMEWORK_VERSION); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Rob Springer "); +module_init(gasket_init); +module_exit(gasket_exit); + +/* + * Perform a standard Gasket callback. + * @gasket_dev: Device specific pointer to forward. + * @cb_function: Standard callback to perform. + */ +static inline int check_and_invoke_callback( + struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) +{ + int ret = 0; + + gasket_nodev_error("check_and_invoke_callback %p", gasket_dev); + if (cb_function) { + mutex_lock(&gasket_dev->mutex); + ret = cb_function(gasket_dev); + mutex_unlock(&gasket_dev->mutex); + } + return ret; +} + +/* + * Perform a standard Gasket callback + * without grabbing gasket_dev->mutex. + * @gasket_dev: Device specific pointer to forward. + * @cb_function: Standard callback to perform. + * + */ +static inline int gasket_check_and_invoke_callback_nolock( + struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) +{ + int ret = 0; + + if (cb_function) { + gasket_log_info( + gasket_dev, "Invoking device-specific callback."); + ret = cb_function(gasket_dev); + } + return ret; +} + +/* + * Retrieve device-specific data via cdev pointer. + * @cdev_ptr: Character device pointer associated with the device. + * + * This function returns the pointer to the device-specific data allocated in + * add_dev_cb for the device associated with cdev_ptr. + */ +static struct gasket_cdev_info *gasket_cdev_get_info(struct cdev *cdev_ptr) +{ + return container_of(cdev_ptr, struct gasket_cdev_info, cdev); +} + +/* + * Returns nonzero if the gasket_cdev_info is owned by the current thread group + * ID. + * @info: Device node info. + */ +static int gasket_owned_by_current_tgid(struct gasket_cdev_info *info) +{ + return (info->ownership.is_owned && + (info->ownership.owner == current->tgid)); +} + +static int __init gasket_init(void) +{ + int i; + + gasket_nodev_info("Performing one-time init of the Gasket framework."); + /* Check for duplicates and find a free slot. */ + mutex_lock(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + g_descs[i].driver_desc = NULL; + mutex_init(&g_descs[i].mutex); + } + + gasket_sysfs_init(); + + mutex_unlock(&g_mutex); + return 0; +} + +static void __exit gasket_exit(void) +{ + /* No deinit/dealloc needed at present. */ + gasket_nodev_info("Removing Gasket framework module."); +} + +/* See gasket_core.h for description. */ +int gasket_register_device(const struct gasket_driver_desc *driver_desc) +{ + int i, ret; + int desc_idx = -1; + struct gasket_internal_desc *internal; + + gasket_nodev_info("Initializing Gasket framework device"); + /* Check for duplicates and find a free slot. */ + mutex_lock(&g_mutex); + + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc == driver_desc) { + gasket_nodev_error( + "%s driver already loaded/registered", + driver_desc->name); + mutex_unlock(&g_mutex); + return -EBUSY; + } + } + + /* This and the above loop could be combined, but this reads easier. */ + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (!g_descs[i].driver_desc) { + g_descs[i].driver_desc = driver_desc; + desc_idx = i; + break; + } + } + mutex_unlock(&g_mutex); + + gasket_nodev_info("Loaded %s driver, framework version %s", + driver_desc->name, GASKET_FRAMEWORK_VERSION); + + if (desc_idx == -1) { + gasket_nodev_error("Too many Gasket drivers loaded: %d\n", + GASKET_FRAMEWORK_DESC_MAX); + return -EBUSY; + } + + /* Internal structure setup. */ + gasket_nodev_info("Performing initial internal structure setup."); + internal = &g_descs[desc_idx]; + mutex_init(&internal->mutex); + memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX); + memset(&internal->pci, 0, sizeof(internal->pci)); + internal->pci.name = driver_desc->name; + internal->pci.id_table = driver_desc->pci_id_table; + internal->pci.probe = gasket_pci_probe; + internal->pci.remove = gasket_pci_remove; + internal->class = + class_create(driver_desc->module, driver_desc->name); + + if (IS_ERR_OR_NULL(internal->class)) { + gasket_nodev_error("Cannot register %s class [ret=%ld]", + driver_desc->name, PTR_ERR(internal->class)); + return PTR_ERR(internal->class); + } + + /* + * Not using pci_register_driver() (without underscores), as it + * depends on KBUILD_MODNAME, and this is a shared file. + */ + gasket_nodev_info("Registering PCI driver."); + ret = __pci_register_driver( + &internal->pci, driver_desc->module, driver_desc->name); + if (ret) { + gasket_nodev_error( + "cannot register pci driver [ret=%d]", ret); + goto fail1; + } + + gasket_nodev_info("Registering char driver."); + ret = register_chrdev_region( + MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, + driver_desc->name); + if (ret) { + gasket_nodev_error("cannot register char driver [ret=%d]", ret); + goto fail2; + } + + gasket_nodev_info("Driver registered successfully."); + return 0; + +fail2: + pci_unregister_driver(&internal->pci); + +fail1: + class_destroy(internal->class); + + g_descs[desc_idx].driver_desc = NULL; + return ret; +} +EXPORT_SYMBOL(gasket_register_device); + +/* See gasket_core.h for description. */ +void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) +{ + int i, desc_idx; + struct gasket_internal_desc *internal_desc = NULL; + + mutex_lock(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc == driver_desc) { + internal_desc = &g_descs[i]; + desc_idx = i; + break; + } + } + mutex_unlock(&g_mutex); + + if (!internal_desc) { + gasket_nodev_error( + "request to unregister unknown desc: %s, %d:%d", + driver_desc->name, driver_desc->major, + driver_desc->minor); + return; + } + + unregister_chrdev_region( + MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX); + + pci_unregister_driver(&internal_desc->pci); + + class_destroy(internal_desc->class); + + /* Finally, effectively "remove" the driver. */ + g_descs[desc_idx].driver_desc = NULL; + + gasket_nodev_info("removed %s driver", driver_desc->name); +} +EXPORT_SYMBOL(gasket_unregister_device); + +/** + * Allocate a Gasket device. + * @internal_desc: Pointer to the internal data for the device driver. + * @pdev: Pointer to the Gasket device pointer, the allocated device. + * @kobj_name: PCIe name for the device + * + * Description: Allocates and initializes a Gasket device structure. + * Adds the device to the device list. + * + * Returns 0 if successful, a negative error code otherwise. + */ +static int gasket_alloc_dev( + struct gasket_internal_desc *internal_desc, struct device *parent, + struct gasket_dev **pdev, const char *kobj_name) +{ + int dev_idx; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + struct gasket_dev *gasket_dev; + struct gasket_cdev_info *dev_info; + + gasket_nodev_info("Allocating a Gasket device %s.", kobj_name); + + *pdev = NULL; + + dev_idx = gasket_find_dev_slot(internal_desc, kobj_name); + if (dev_idx < 0) + return dev_idx; + + gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL); + if (!gasket_dev) { + gasket_nodev_error("no memory for device"); + return -ENOMEM; + } + internal_desc->devs[dev_idx] = gasket_dev; + + mutex_init(&gasket_dev->mutex); + + gasket_dev->internal_desc = internal_desc; + gasket_dev->dev_idx = dev_idx; + snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", kobj_name); + /* gasket_bar_data is uninitialized. */ + gasket_dev->num_page_tables = driver_desc->num_page_tables; + /* max_page_table_size and *page table are uninit'ed */ + /* interrupt_data is not initialized. */ + /* status is 0, or GASKET_STATUS_DEAD */ + + dev_info = &gasket_dev->dev_info; + snprintf(dev_info->name, GASKET_NAME_MAX, "%s_%u", driver_desc->name, + gasket_dev->dev_idx); + dev_info->devt = + MKDEV(driver_desc->major, driver_desc->minor + + gasket_dev->dev_idx); + dev_info->device = device_create(internal_desc->class, parent, + dev_info->devt, gasket_dev, dev_info->name); + + gasket_nodev_info("Gasket device allocated: %p.", dev_info->device); + + /* cdev has not yet been added; cdev_added is 0 */ + dev_info->gasket_dev_ptr = gasket_dev; + /* ownership is all 0, indicating no owner or opens. */ + + return 0; +} + +/* + * Free a Gasket device. + * @internal_dev: Gasket device pointer; the device to unregister and free. + * + * Description: Removes the device from the device list and frees + * the Gasket device structure. + */ +static void gasket_free_dev(struct gasket_dev *gasket_dev) +{ + struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc; + + mutex_lock(&internal_desc->mutex); + internal_desc->devs[gasket_dev->dev_idx] = NULL; + mutex_unlock(&internal_desc->mutex); + + kfree(gasket_dev); +} + +/* + * Finds the next free gasket_internal_dev slot. + * + * Returns the located slot number on success or a negative number on failure. + */ +static int gasket_find_dev_slot( + struct gasket_internal_desc *internal_desc, const char *kobj_name) +{ + int i; + + mutex_lock(&internal_desc->mutex); + + /* Search for a previous instance of this device. */ + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (internal_desc->devs[i] && + strcmp(internal_desc->devs[i]->kobj_name, kobj_name) == 0) { + gasket_nodev_error("Duplicate device %s", kobj_name); + mutex_unlock(&internal_desc->mutex); + return -EBUSY; + } + } + + /* Find a free device slot. */ + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (!internal_desc->devs[i]) + break; + } + + if (i == GASKET_DEV_MAX) { + gasket_nodev_info( + "Too many registered devices; max %d", GASKET_DEV_MAX); + mutex_unlock(&internal_desc->mutex); + return -EBUSY; + } + + mutex_unlock(&internal_desc->mutex); + return i; +} + +/** + * PCI subsystem probe function. + * @pci_dev: PCI device pointer to the new device. + * @id: PCI device id structure pointer, the vendor and device ids. + * + * Called when a Gasket device is found. Allocates device metadata, maps device + * memory, and calls gasket_enable_dev to prepare the device for active use. + * + * Returns 0 if successful and a negative value otherwise. + */ +static int gasket_pci_probe( + struct pci_dev *pci_dev, const struct pci_device_id *id) +{ + int ret; + const char *kobj_name = dev_name(&pci_dev->dev); + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc; + struct device *parent; + + gasket_nodev_info("Add Gasket device %s", kobj_name); + + mutex_lock(&g_mutex); + internal_desc = lookup_internal_desc(pci_dev); + mutex_unlock(&g_mutex); + if (!internal_desc) { + gasket_nodev_info("PCI probe called for unknown driver type"); + return -ENODEV; + } + + driver_desc = internal_desc->driver_desc; + + parent = &pci_dev->dev; + ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); + if (ret) + return ret; + if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { + gasket_nodev_error("Cannot create %s device %s [ret = %ld]", + driver_desc->name, gasket_dev->dev_info.name, + PTR_ERR(gasket_dev->dev_info.device)); + ret = -ENODEV; + goto fail1; + } + gasket_dev->pci_dev = pci_dev; + + ret = gasket_setup_pci(pci_dev, gasket_dev); + if (ret) + goto fail2; + + ret = check_and_invoke_callback(gasket_dev, driver_desc->add_dev_cb); + if (ret) { + gasket_log_error(gasket_dev, "Error in add device cb: %d", ret); + goto fail2; + } + + ret = gasket_sysfs_create_mapping( + gasket_dev->dev_info.device, gasket_dev); + if (ret) + goto fail3; + + /* + * Once we've created the mapping structures successfully, attempt to + * create a symlink to the pci directory of this object. + */ + ret = sysfs_create_link(&gasket_dev->dev_info.device->kobj, + &pci_dev->dev.kobj, dev_name(&pci_dev->dev)); + if (ret) { + gasket_log_error( + gasket_dev, "Cannot create sysfs pci link: %d", ret); + goto fail3; + } + ret = gasket_sysfs_create_entries( + gasket_dev->dev_info.device, gasket_sysfs_generic_attrs); + if (ret) + goto fail4; + + ret = check_and_invoke_callback( + gasket_dev, driver_desc->sysfs_setup_cb); + if (ret) { + gasket_log_error( + gasket_dev, "Error in sysfs setup cb: %d", ret); + goto fail5; + } + + ret = gasket_enable_dev(internal_desc, gasket_dev); + if (ret) { + gasket_nodev_error("cannot setup %s device", driver_desc->name); + gasket_disable_dev(gasket_dev); + goto fail5; + } + + return 0; + +fail5: + check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); +fail4: +fail3: + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); +fail2: + gasket_cleanup_pci(gasket_dev); + check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); +fail1: + gasket_free_dev(gasket_dev); + return ret; +} + +/* + * PCI subsystem remove function. + * @pci_dev: PCI device pointer; the device to remove. + * + * Called to remove a Gasket device. Finds the device in the device list and + * cleans up metadata. + */ +static void gasket_pci_remove(struct pci_dev *pci_dev) +{ + int i; + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev = NULL; + const struct gasket_driver_desc *driver_desc; + /* Find the device desc. */ + mutex_lock(&g_mutex); + internal_desc = lookup_internal_desc(pci_dev); + if (!internal_desc) { + mutex_unlock(&g_mutex); + return; + } + mutex_unlock(&g_mutex); + + driver_desc = internal_desc->driver_desc; + + /* Now find the specific device */ + mutex_lock(&internal_desc->mutex); + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (internal_desc->devs[i] && + internal_desc->devs[i]->pci_dev == pci_dev) { + gasket_dev = internal_desc->devs[i]; + break; + } + } + mutex_unlock(&internal_desc->mutex); + + if (!gasket_dev) + return; + + gasket_nodev_info( + "remove %s device %s", internal_desc->driver_desc->name, + gasket_dev->kobj_name); + + gasket_disable_dev(gasket_dev); + gasket_cleanup_pci(gasket_dev); + + check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); + + check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); + + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); + gasket_free_dev(gasket_dev); +} + +/* + * Setup PCI & set up memory mapping for the specified device. + * @pci_dev: pointer to the particular PCI device. + * @internal_dev: Corresponding Gasket device pointer. + * + * Enables the PCI device, reads the BAR registers and sets up pointers to the + * device's memory mapped IO space. + * + * Returns 0 on success and a negative value otherwise. + */ +static int gasket_setup_pci( + struct pci_dev *pci_dev, struct gasket_dev *gasket_dev) +{ + int i, mapped_bars, ret; + + gasket_dev->pci_dev = pci_dev; + ret = pci_enable_device(pci_dev); + if (ret) { + gasket_log_error(gasket_dev, "cannot enable PCI device"); + return ret; + } + + pci_set_master(pci_dev); + + for (i = 0; i < GASKET_NUM_BARS; i++) { + ret = gasket_map_pci_bar(gasket_dev, i); + if (ret) { + mapped_bars = i; + goto fail; + } + } + + return 0; + +fail: + for (i = 0; i < mapped_bars; i++) + gasket_unmap_pci_bar(gasket_dev, i); + + pci_disable_device(pci_dev); + return -ENOMEM; +} + +/* Unmaps memory and cleans up PCI for the specified device. */ +static void gasket_cleanup_pci(struct gasket_dev *gasket_dev) +{ + int i; + + for (i = 0; i < GASKET_NUM_BARS; i++) + gasket_unmap_pci_bar(gasket_dev, i); + + pci_disable_device(gasket_dev->pci_dev); +} + +/* + * Maps the specified bar into kernel space. + * @internal_dev: Device possessing the BAR to map. + * @bar_num: The BAR to map. + * + * Returns 0 on success, a negative error code otherwise. + * A zero-sized BAR will not be mapped, but is not an error. + */ +static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) +{ + struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + ulong desc_bytes = driver_desc->bar_descriptions[bar_num].size; + int ret; + + if (desc_bytes == 0) + return 0; + + if (driver_desc->bar_descriptions[bar_num].type != PCI_BAR) { + /* not PCI: skip this entry */ + return 0; + } + /* + * pci_resource_start and pci_resource_len return a "resource_size_t", + * which is safely castable to ulong (which itself is the arg to + * request_mem_region). + */ + gasket_dev->bar_data[bar_num].phys_base = + (ulong)pci_resource_start(gasket_dev->pci_dev, bar_num); + if (!gasket_dev->bar_data[bar_num].phys_base) { + gasket_log_error(gasket_dev, "Cannot get BAR%u base address", + bar_num); + return -EINVAL; + } + + gasket_dev->bar_data[bar_num].length_bytes = + (ulong)pci_resource_len(gasket_dev->pci_dev, bar_num); + if (gasket_dev->bar_data[bar_num].length_bytes < desc_bytes) { + gasket_log_error( + gasket_dev, + "PCI BAR %u space is too small: %lu; expected >= %lu", + bar_num, gasket_dev->bar_data[bar_num].length_bytes, + desc_bytes); + return -ENOMEM; + } + + if (!request_mem_region(gasket_dev->bar_data[bar_num].phys_base, + gasket_dev->bar_data[bar_num].length_bytes, + gasket_dev->dev_info.name)) { + gasket_log_error( + gasket_dev, + "Cannot get BAR %d memory region %p", + bar_num, &gasket_dev->pci_dev->resource[bar_num]); + return -EINVAL; + } + + gasket_dev->bar_data[bar_num].virt_base = + ioremap_nocache(gasket_dev->bar_data[bar_num].phys_base, + gasket_dev->bar_data[bar_num].length_bytes); + if (!gasket_dev->bar_data[bar_num].virt_base) { + gasket_log_error( + gasket_dev, + "Cannot remap BAR %d memory region %p", + bar_num, &gasket_dev->pci_dev->resource[bar_num]); + ret = -ENOMEM; + goto fail; + } + + dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64)); + dma_set_coherent_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64)); + + return 0; + +fail: + iounmap(gasket_dev->bar_data[bar_num].virt_base); + release_mem_region(gasket_dev->bar_data[bar_num].phys_base, + gasket_dev->bar_data[bar_num].length_bytes); + return ret; +} + +/* + * Releases PCI BAR mapping. + * @internal_dev: Device possessing the BAR to unmap. + * + * A zero-sized or not-mapped BAR will not be unmapped, but is not an error. + */ +static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num) +{ + ulong base, bytes; + struct gasket_internal_desc *internal_desc = dev->internal_desc; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + + if (driver_desc->bar_descriptions[bar_num].size == 0 || + !dev->bar_data[bar_num].virt_base) + return; + + if (driver_desc->bar_descriptions[bar_num].type != PCI_BAR) + return; + + iounmap(dev->bar_data[bar_num].virt_base); + dev->bar_data[bar_num].virt_base = NULL; + + base = pci_resource_start(dev->pci_dev, bar_num); + if (!base) { + gasket_log_error( + dev, "cannot get PCI BAR%u base address", bar_num); + return; + } + + bytes = pci_resource_len(dev->pci_dev, bar_num); + release_mem_region(base, bytes); +} + +/* + * Handle adding a char device and related info. + * @dev_info: Pointer to the dev_info struct for this device. + * @file_ops: The file operations for this device. + * @owner: The owning module for this device. + */ +static int gasket_add_cdev( + struct gasket_cdev_info *dev_info, + const struct file_operations *file_ops, struct module *owner) +{ + int ret; + + cdev_init(&dev_info->cdev, file_ops); + dev_info->cdev.owner = owner; + ret = cdev_add(&dev_info->cdev, dev_info->devt, 1); + if (ret) { + gasket_log_error( + dev_info->gasket_dev_ptr, + "cannot add char device [ret=%d]", ret); + return ret; + } + dev_info->cdev_added = 1; + + return 0; +} + +/* + * Performs final init and marks the device as active. + * @internal_desc: Pointer to Gasket [internal] driver descriptor structure. + * @internal_dev: Pointer to Gasket [internal] device structure. + * + * Currently forwards all work to device-specific callback; a future phase will + * extract elements of character device registration here. + */ +static int gasket_enable_dev( + struct gasket_internal_desc *internal_desc, + struct gasket_dev *gasket_dev) +{ + int tbl_idx; + int ret; + bool has_dma_ops; + struct device *ddev; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + + ret = gasket_interrupt_init( + gasket_dev, driver_desc->name, + driver_desc->interrupt_type, driver_desc->interrupts, + driver_desc->num_interrupts, driver_desc->interrupt_pack_width, + driver_desc->interrupt_bar_index, + driver_desc->wire_interrupt_offsets); + if (ret) { + gasket_log_error(gasket_dev, + "Critical failure to allocate interrupts: %d", + ret); + gasket_interrupt_cleanup(gasket_dev); + return ret; + } + + has_dma_ops = true; + + for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { + gasket_log_debug( + gasket_dev, "Initializing page table %d.", tbl_idx); + if (gasket_dev->pci_dev) { + ddev = &gasket_dev->pci_dev->dev; + } else { + gasket_log_error( + gasket_dev, + "gasket_enable_dev with no physical device!!"); + WARN_ON(1); + ddev = NULL; + } + ret = gasket_page_table_init( + &gasket_dev->page_table[tbl_idx], + &gasket_dev->bar_data[ + driver_desc->page_table_bar_index], + &driver_desc->page_table_configs[tbl_idx], + ddev, gasket_dev->pci_dev, has_dma_ops); + if (ret) { + gasket_log_error( + gasket_dev, + "Couldn't init page table %d: %d", + tbl_idx, ret); + return ret; + } + /* + * Make sure that the page table is clear and set to simple + * addresses. + */ + gasket_page_table_reset(gasket_dev->page_table[tbl_idx]); + } + + /* + * hardware_revision_cb returns a positive integer (the rev) if + * successful.) + */ + ret = check_and_invoke_callback( + gasket_dev, driver_desc->hardware_revision_cb); + if (ret < 0) { + gasket_log_error( + gasket_dev, "Error getting hardware revision: %d", ret); + return ret; + } + gasket_dev->hardware_revision = ret; + + ret = check_and_invoke_callback(gasket_dev, driver_desc->enable_dev_cb); + if (ret) { + gasket_log_error( + gasket_dev, "Error in enable device cb: %d", ret); + return ret; + } + + /* device_status_cb returns a device status, not an error code. */ + gasket_dev->status = gasket_get_hw_status(gasket_dev); + if (gasket_dev->status == GASKET_STATUS_DEAD) + gasket_log_error(gasket_dev, "Device reported as unhealthy."); + + ret = gasket_add_cdev( + &gasket_dev->dev_info, &gasket_file_ops, driver_desc->module); + if (ret) + return ret; + + return 0; +} + +/* + * Disable device operations. + * @gasket_dev: Pointer to Gasket device structure. + * + * Currently forwards all work to device-specific callback; a future phase will + * extract elements of character device unregistration here. + */ +static void gasket_disable_dev(struct gasket_dev *gasket_dev) +{ + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + int i; + + /* Only delete the device if it has been successfully added. */ + if (gasket_dev->dev_info.cdev_added) + cdev_del(&gasket_dev->dev_info.cdev); + + gasket_dev->status = GASKET_STATUS_DEAD; + + gasket_interrupt_cleanup(gasket_dev); + + for (i = 0; i < driver_desc->num_page_tables; ++i) { + if (gasket_dev->page_table[i]) { + gasket_page_table_reset(gasket_dev->page_table[i]); + gasket_page_table_cleanup(gasket_dev->page_table[i]); + } + } + + check_and_invoke_callback(gasket_dev, driver_desc->disable_dev_cb); +} + +/** + * Registered descriptor lookup. + * + * Precondition: Called with g_mutex held (to avoid a race on return). + * Returns NULL if no matching device was found. + */ +static struct gasket_internal_desc *lookup_internal_desc( + struct pci_dev *pci_dev) +{ + int i; + + __must_hold(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc && + g_descs[i].driver_desc->pci_id_table && + pci_match_id(g_descs[i].driver_desc->pci_id_table, pci_dev)) + return &g_descs[i]; + } + + return NULL; +} + +/** + * Lookup a name by number in a num_name table. + * @num: Number to lookup. + * @table: Array of num_name structures, the table for the lookup. + * + * Description: Searches for num in the table. If found, the + * corresponding name is returned; otherwise NULL + * is returned. + * + * The table must have a NULL name pointer at the end. + */ +const char *gasket_num_name_lookup( + uint num, const struct gasket_num_name *table) +{ + uint i = 0; + + while (table[i].snn_name) { + if (num == table[i].snn_num) + break; + ++i; + } + + return table[i].snn_name; +} +EXPORT_SYMBOL(gasket_num_name_lookup); + +/** + * Opens the char device file. + * @inode: Inode structure pointer of the device file. + * @file: File structure pointer. + * + * Description: Called on an open of the device file. If the open is for + * writing, and the device is not owned, this process becomes + * the owner. If the open is for writing and the device is + * already owned by some other process, it is an error. If + * this process is the owner, increment the open count. + * + * Returns 0 if successful, a negative error number otherwise. + */ +static int gasket_open(struct inode *inode, struct file *filp) +{ + int ret; + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc; + struct gasket_ownership *ownership; + char task_name[TASK_COMM_LEN]; + struct gasket_cdev_info *dev_info = gasket_cdev_get_info(inode->i_cdev); + + if (!dev_info) { + gasket_nodev_error("Unable to retrieve device data"); + return -EINVAL; + } + gasket_dev = dev_info->gasket_dev_ptr; + driver_desc = gasket_dev->internal_desc->driver_desc; + ownership = &dev_info->ownership; + get_task_comm(task_name, current); + filp->private_data = gasket_dev; + inode->i_size = 0; + + gasket_log_debug( + gasket_dev, + "Attempting to open with tgid %u (%s) (f_mode: 0%03o, " + "fmode_write: %d is_root: %u)", + current->tgid, task_name, filp->f_mode, + (filp->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN)); + + /* Always allow non-writing accesses. */ + if (!(filp->f_mode & FMODE_WRITE)) { + gasket_log_debug(gasket_dev, "Allowing read-only opening."); + return 0; + } + + mutex_lock(&gasket_dev->mutex); + + gasket_log_debug( + gasket_dev, "Current owner open count (owning tgid %u): %d.", + ownership->owner, ownership->write_open_count); + + /* Opening a node owned by another TGID is an error (even root.) */ + if (ownership->is_owned && ownership->owner != current->tgid) { + gasket_log_error( + gasket_dev, + "Process %u is opening a node held by %u.", + current->tgid, ownership->owner); + mutex_unlock(&gasket_dev->mutex); + return -EPERM; + } + + /* If the node is not owned, assign it to the current TGID. */ + if (!ownership->is_owned) { + ret = gasket_check_and_invoke_callback_nolock( + gasket_dev, driver_desc->device_open_cb); + if (ret) { + gasket_log_error( + gasket_dev, "Error in device open cb: %d", ret); + mutex_unlock(&gasket_dev->mutex); + return ret; + } + ownership->is_owned = 1; + ownership->owner = current->tgid; + gasket_log_debug(gasket_dev, "Device owner is now tgid %u", + ownership->owner); + } + + ownership->write_open_count++; + + gasket_log_debug(gasket_dev, "New open count (owning tgid %u): %d", + ownership->owner, ownership->write_open_count); + + mutex_unlock(&gasket_dev->mutex); + return 0; +} + +/** + * gasket_release - Close of the char device file. + * @inode: Inode structure pointer of the device file. + * @file: File structure pointer. + * + * Description: Called on a close of the device file. If this process + * is the owner, decrement the open count. On last close + * by the owner, free up buffers and eventfd contexts, and + * release ownership. + * + * Returns 0 if successful, a negative error number otherwise. + */ +static int gasket_release(struct inode *inode, struct file *file) +{ + int i; + struct gasket_dev *gasket_dev; + struct gasket_ownership *ownership; + const struct gasket_driver_desc *driver_desc; + char task_name[TASK_COMM_LEN]; + struct gasket_cdev_info *dev_info = + (struct gasket_cdev_info *)gasket_cdev_get_info(inode->i_cdev); + if (!dev_info) { + gasket_nodev_error("Unable to retrieve device data"); + return -EINVAL; + } + gasket_dev = dev_info->gasket_dev_ptr; + driver_desc = gasket_dev->internal_desc->driver_desc; + ownership = &dev_info->ownership; + get_task_comm(task_name, current); + mutex_lock(&gasket_dev->mutex); + + gasket_log_debug( + gasket_dev, + "Releasing device node. Call origin: tgid %u (%s) " + "(f_mode: 0%03o, fmode_write: %d, is_root: %u)", + current->tgid, task_name, file->f_mode, + (file->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN)); + gasket_log_debug(gasket_dev, "Current open count (owning tgid %u): %d", + ownership->owner, ownership->write_open_count); + + if (file->f_mode & FMODE_WRITE) { + ownership->write_open_count--; + if (ownership->write_open_count == 0) { + gasket_log_info(gasket_dev, "Device is now free"); + ownership->is_owned = 0; + ownership->owner = 0; + + /* Forces chip reset before we unmap the page tables. */ + driver_desc->device_reset_cb(gasket_dev, 0); + + for (i = 0; i < driver_desc->num_page_tables; ++i) { + gasket_page_table_unmap_all( + gasket_dev->page_table[i]); + gasket_page_table_garbage_collect( + gasket_dev->page_table[i]); + gasket_free_coherent_memory_all(gasket_dev, i); + } + + /* Closes device, enters power save. */ + gasket_check_and_invoke_callback_nolock( + gasket_dev, driver_desc->device_close_cb); + } + } + + gasket_log_info( + gasket_dev, "New open count (owning tgid %u): %d", + ownership->owner, ownership->write_open_count); + mutex_unlock(&gasket_dev->mutex); + return 0; +} + +/* + * Permission and validity checking for mmap ops. + * @gasket_dev: Gasket device information structure. + * @vma: Standard virtual memory area descriptor. + * + * Verifies that the user has permissions to perform the requested mapping and + * that the provided descriptor/range is of adequate size to hold the range to + * be mapped. + */ +static int gasket_mmap_has_permissions( + struct gasket_dev *gasket_dev, struct vm_area_struct *vma, + int bar_permissions) +{ + int requested_permissions; + /* Always allow sysadmin to access. */ + if (capable(CAP_SYS_ADMIN)) + return 1; + + /* Never allow non-sysadmins to access to a dead device. */ + if (gasket_dev->status != GASKET_STATUS_ALIVE) { + gasket_log_info(gasket_dev, "Device is dead."); + return 0; + } + + /* Make sure that no wrong flags are set. */ + requested_permissions = + (vma->vm_flags & (VM_WRITE | VM_READ | VM_EXEC)); + if (requested_permissions & ~(bar_permissions)) { + gasket_log_info( + gasket_dev, + "Attempting to map a region with requested permissions " + "0x%x, but region has permissions 0x%x.", + requested_permissions, bar_permissions); + return 0; + } + + /* Do not allow a non-owner to write. */ + if ((vma->vm_flags & VM_WRITE) && + !gasket_owned_by_current_tgid(&gasket_dev->dev_info)) { + gasket_log_info( + gasket_dev, + "Attempting to mmap a region for write without owning " + "device."); + return 0; + } + + return 1; +} + +/* + * Checks if an address is within the region + * allocated for coherent buffer. + * @driver_desc: driver description. + * @address: offset of address to check. + * + * Verifies that the input address is within the region allocated to coherent + * buffer. + */ +static bool gasket_is_coherent_region( + const struct gasket_driver_desc *driver_desc, ulong address) +{ + struct gasket_coherent_buffer_desc coh_buff_desc = + driver_desc->coherent_buffer_description; + + if (coh_buff_desc.permissions != GASKET_NOMAP) { + if ((address >= coh_buff_desc.base) && + (address < coh_buff_desc.base + coh_buff_desc.size)) { + return true; + } + } + return false; +} + +static int gasket_get_bar_index( + const struct gasket_dev *gasket_dev, ulong phys_addr) +{ + int i; + const struct gasket_driver_desc *driver_desc; + + driver_desc = gasket_dev->internal_desc->driver_desc; + for (i = 0; i < GASKET_NUM_BARS; ++i) { + struct gasket_bar_desc bar_desc = + driver_desc->bar_descriptions[i]; + + if (bar_desc.permissions != GASKET_NOMAP) { + if (phys_addr >= bar_desc.base && + phys_addr < (bar_desc.base + bar_desc.size)) { + return i; + } + } + } + /* If we haven't found the address by now, it is invalid. */ + return -EINVAL; +} + +/* + * Sets the actual bounds to map, given the device's mappable region. + * + * Given the device's mappable region, along with the user-requested mapping + * start offset and length of the user region, determine how much of this + * mappable region can be mapped into the user's region (start/end offsets), + * and the physical offset (phys_offset) into the BAR where the mapping should + * begin (either the VMA's or region lower bound). + * + * In other words, this calculates the overlap between the VMA + * (bar_offset, requested_length) and the given gasket_mappable_region. + * + * Returns true if there's anything to map, and false otherwise. + */ +static bool gasket_mm_get_mapping_addrs( + const struct gasket_mappable_region *region, ulong bar_offset, + ulong requested_length, struct gasket_mappable_region *mappable_region, + ulong *virt_offset) +{ + ulong range_start = region->start; + ulong range_length = region->length_bytes; + ulong range_end = range_start + range_length; + + *virt_offset = 0; + if (bar_offset + requested_length < range_start) { + /* + * If the requested region is completely below the range, + * there is nothing to map. + */ + return false; + } else if (bar_offset <= range_start) { + /* If the bar offset is below this range's start + * but the requested length continues into it: + * 1) Only map starting from the beginning of this + * range's phys. offset, so we don't map unmappable + * memory. + * 2) The length of the virtual memory to not map is the + * delta between the bar offset and the + * mappable start (and since the mappable start is + * bigger, start - req.) + * 3) The map length is the minimum of the mappable + * requested length (requested_length - virt_offset) + * and the actual mappable length of the range. + */ + mappable_region->start = range_start; + *virt_offset = range_start - bar_offset; + mappable_region->length_bytes = + min(requested_length - *virt_offset, range_length); + return true; + } else if (bar_offset > range_start && + bar_offset < range_end) { + /* + * If the bar offset is within this range: + * 1) Map starting from the bar offset. + * 2) Because there is no forbidden memory between the + * bar offset and the range start, + * virt_offset is 0. + * 3) The map length is the minimum of the requested + * length and the remaining length in the buffer + * (range_end - bar_offset) + */ + mappable_region->start = bar_offset; + *virt_offset = 0; + mappable_region->length_bytes = min( + requested_length, range_end - bar_offset); + return true; + } + + /* + * If the requested [start] offset is above range_end, + * there's nothing to map. + */ + return false; +} + +int gasket_mm_unmap_region( + const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, + const struct gasket_mappable_region *map_region) +{ + ulong bar_offset; + ulong virt_offset; + struct gasket_mappable_region mappable_region; + int ret; + + if (map_region->length_bytes == 0) + return 0; + + ret = gasket_mm_vma_bar_offset(gasket_dev, vma, &bar_offset); + if (ret) + return ret; + + if (!gasket_mm_get_mapping_addrs( + map_region, bar_offset, vma->vm_end - vma->vm_start, + &mappable_region, &virt_offset)) + return 1; + + /* + * The length passed to zap_vma_ptes MUST BE A MULTIPLE OF + * PAGE_SIZE! Trust me. I have the scars. + * + * Next multiple of y: ceil_div(x, y) * y + */ + zap_vma_ptes( + vma, vma->vm_start + virt_offset, + DIV_ROUND_UP(mappable_region.length_bytes, PAGE_SIZE) * + PAGE_SIZE); + return 0; +} +EXPORT_SYMBOL(gasket_mm_unmap_region); + +/* Maps a virtual address + range to a physical offset of a BAR. */ +static enum do_map_region_status do_map_region( + const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, + struct gasket_mappable_region *mappable_region) +{ + /* Maximum size of a single call to io_remap_pfn_range. */ + /* I pulled this number out of thin air. */ + const ulong max_chunk_size = 64 * 1024 * 1024; + ulong chunk_size, mapped_bytes = 0; + + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + + ulong bar_offset, virt_offset; + struct gasket_mappable_region region_to_map; + ulong phys_offset, map_length; + ulong virt_base, phys_base; + int bar_index, ret; + + ret = gasket_mm_vma_bar_offset(gasket_dev, vma, &bar_offset); + if (ret) + return DO_MAP_REGION_INVALID; + + if (!gasket_mm_get_mapping_addrs(mappable_region, bar_offset, + vma->vm_end - vma->vm_start, + ®ion_to_map, &virt_offset)) + return DO_MAP_REGION_INVALID; + phys_offset = region_to_map.start; + map_length = region_to_map.length_bytes; + + virt_base = vma->vm_start + virt_offset; + bar_index = + gasket_get_bar_index( + gasket_dev, + (vma->vm_pgoff << PAGE_SHIFT) + + driver_desc->legacy_mmap_address_offset); + phys_base = gasket_dev->bar_data[bar_index].phys_base + phys_offset; + while (mapped_bytes < map_length) { + /* + * io_remap_pfn_range can take a while, so we chunk its + * calls and call cond_resched between each. + */ + chunk_size = min(max_chunk_size, map_length - mapped_bytes); + + cond_resched(); + ret = io_remap_pfn_range( + vma, virt_base + mapped_bytes, + (phys_base + mapped_bytes) >> PAGE_SHIFT, + chunk_size, vma->vm_page_prot); + if (ret) { + gasket_log_error( + gasket_dev, "Error remapping PFN range."); + goto fail; + } + mapped_bytes += chunk_size; + } + + return DO_MAP_REGION_SUCCESS; + +fail: + /* Unmap the partial chunk we mapped. */ + mappable_region->length_bytes = mapped_bytes; + if (gasket_mm_unmap_region(gasket_dev, vma, mappable_region)) + gasket_log_error( + gasket_dev, + "Error unmapping partial region 0x%lx (0x%lx bytes)", + (ulong)virt_offset, + (ulong)mapped_bytes); + + return DO_MAP_REGION_FAILURE; +} + +/* + * Calculates the offset where the VMA range begins in its containing BAR. + * The offset is written into bar_offset on success. + * Returns zero on success, anything else on error. +*/ +static int gasket_mm_vma_bar_offset( + const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, + ulong *bar_offset) +{ + ulong raw_offset; + int bar_index; + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + + raw_offset = (vma->vm_pgoff << PAGE_SHIFT) + + driver_desc->legacy_mmap_address_offset; + bar_index = gasket_get_bar_index(gasket_dev, raw_offset); + if (bar_index < 0) { + gasket_log_error( + gasket_dev, + "Unable to find matching bar for address 0x%lx", + raw_offset); + trace_gasket_mmap_exit(bar_index); + return bar_index; + } + *bar_offset = + raw_offset - driver_desc->bar_descriptions[bar_index].base; + + return 0; +} + +/* + * Map a region of coherent memory. + * @gasket_dev: Gasket device handle. + * @vma: Virtual memory area descriptor with region to map. + */ +static int gasket_mmap_coherent( + struct gasket_dev *gasket_dev, struct vm_area_struct *vma) +{ + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + const ulong requested_length = vma->vm_end - vma->vm_start; + int ret; + ulong permissions; + + if (requested_length == 0 || requested_length > + gasket_dev->coherent_buffer.length_bytes) { + trace_gasket_mmap_exit(-EINVAL); + return -EINVAL; + } + + permissions = driver_desc->coherent_buffer_description.permissions; + if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) { + gasket_log_error(gasket_dev, "Permission checking failed."); + trace_gasket_mmap_exit(-EPERM); + return -EPERM; + } + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + ret = remap_pfn_range( + vma, vma->vm_start, + (gasket_dev->coherent_buffer.phys_base) >> PAGE_SHIFT, + requested_length, vma->vm_page_prot); + if (ret) { + gasket_log_error( + gasket_dev, "Error remapping PFN range err=%d.", ret); + trace_gasket_mmap_exit(ret); + return ret; + } + + /* Record the user virtual to dma_address mapping that was + * created by the kernel. + */ + gasket_set_user_virt( + gasket_dev, requested_length, + gasket_dev->coherent_buffer.phys_base, vma->vm_start); + return 0; +} + +/* + * Maps a device's BARs into user space. + * @filp: File structure pointer describing this node usage session. + * @vma: Standard virtual memory area descriptor. + * + * Maps the entirety of each of the device's BAR ranges into the user memory + * range specified by vma. + * + * Returns 0 on success, a negative errno on error. + */ +static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int i, ret; + int bar_index; + int has_mapped_anything = 0; + ulong permissions; + ulong raw_offset, vma_size; + bool is_coherent_region; + const struct gasket_driver_desc *driver_desc; + struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; + struct gasket_bar_data *bar_data; + const struct gasket_bar_desc *bar_desc; + struct gasket_mappable_region *map_regions = NULL; + int num_map_regions = 0; + enum do_map_region_status map_status; + + if (!gasket_dev) { + gasket_nodev_error("Unable to retrieve device data"); + trace_gasket_mmap_exit(-EINVAL); + return -EINVAL; + } + driver_desc = gasket_dev->internal_desc->driver_desc; + + if (vma->vm_start & (PAGE_SIZE - 1)) { + gasket_log_error( + gasket_dev, "Base address not page-aligned: 0x%p\n", + (void *)vma->vm_start); + trace_gasket_mmap_exit(-EINVAL); + return -EINVAL; + } + + /* Calculate the offset of this range into physical mem. */ + raw_offset = (vma->vm_pgoff << PAGE_SHIFT) + + driver_desc->legacy_mmap_address_offset; + vma_size = vma->vm_end - vma->vm_start; + trace_gasket_mmap_entry( + gasket_dev->dev_info.name, raw_offset, vma_size); + + /* + * Check if the raw offset is within a bar region. If not, check if it + * is a coherent region. + */ + bar_index = gasket_get_bar_index(gasket_dev, raw_offset); + is_coherent_region = gasket_is_coherent_region(driver_desc, raw_offset); + if (bar_index < 0 && !is_coherent_region) { + gasket_log_error( + gasket_dev, + "Unable to find matching bar for address 0x%lx", + raw_offset); + trace_gasket_mmap_exit(bar_index); + return bar_index; + } + if (bar_index > 0 && is_coherent_region) { + gasket_log_error( + gasket_dev, + "double matching bar and coherent buffers for address " + "0x%lx", + raw_offset); + trace_gasket_mmap_exit(bar_index); + return bar_index; + } + + vma->vm_private_data = gasket_dev; + + if (is_coherent_region) + return gasket_mmap_coherent(gasket_dev, vma); + + /* Everything in the rest of this function is for normal BAR mapping. */ + + /* + * Subtract the base of the bar from the raw offset to get the + * memory location within the bar to map. + */ + bar_data = &gasket_dev->bar_data[bar_index]; + + bar_desc = &driver_desc->bar_descriptions[bar_index]; + permissions = bar_desc->permissions; + if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) { + gasket_log_error(gasket_dev, "Permission checking failed."); + trace_gasket_mmap_exit(-EPERM); + return -EPERM; + } + + if (driver_desc->get_mappable_regions_cb) { + ret = driver_desc->get_mappable_regions_cb( + gasket_dev, bar_index, &map_regions, &num_map_regions); + if (ret) + return ret; + } else { + if (!gasket_mmap_has_permissions(gasket_dev, vma, + bar_desc->permissions)) { + gasket_log_error( + gasket_dev, "Permission checking failed."); + trace_gasket_mmap_exit(-EPERM); + return -EPERM; + } + num_map_regions = bar_desc->num_mappable_regions; + map_regions = kzalloc( + num_map_regions * sizeof(*bar_desc->mappable_regions), + GFP_KERNEL); + if (map_regions) { + memcpy(map_regions, bar_desc->mappable_regions, + num_map_regions * + sizeof(*bar_desc->mappable_regions)); + } + } + + if (!map_regions || num_map_regions == 0) { + gasket_log_error(gasket_dev, "No mappable regions returned!"); + return -EINVAL; + } + + /* Marks the VMA's pages as uncacheable. */ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + for (i = 0; i < num_map_regions; i++) { + map_status = do_map_region(gasket_dev, vma, &map_regions[i]); + /* Try the next region if this one was not mappable. */ + if (map_status == DO_MAP_REGION_INVALID) + continue; + if (map_status == DO_MAP_REGION_FAILURE) + goto fail; + + has_mapped_anything = 1; + } + + kfree(map_regions); + + /* If we could not map any memory, the request was invalid. */ + if (!has_mapped_anything) { + gasket_log_error( + gasket_dev, + "Map request did not contain a valid region."); + trace_gasket_mmap_exit(-EINVAL); + return -EINVAL; + } + + trace_gasket_mmap_exit(0); + return 0; + +fail: + /* Need to unmap any mapped ranges. */ + num_map_regions = i; + for (i = 0; i < num_map_regions; i++) + if (gasket_mm_unmap_region(gasket_dev, vma, + &bar_desc->mappable_regions[i])) + gasket_log_error( + gasket_dev, "Error unmapping range %d.", i); + kfree(map_regions); + + return ret; +} + +/* + * Determine the health of the Gasket device. + * @gasket_dev: Gasket device structure. + * + * Checks the underlying device health (via the device_status_cb) + * and the status of initialized Gasket code systems (currently + * only interrupts), then returns a gasket_status appropriately. + */ +static int gasket_get_hw_status(struct gasket_dev *gasket_dev) +{ + int status; + int i; + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + + status = gasket_check_and_invoke_callback_nolock( + gasket_dev, driver_desc->device_status_cb); + if (status != GASKET_STATUS_ALIVE) { + gasket_log_info(gasket_dev, "Hardware reported status %d.", + status); + return status; + } + + status = gasket_interrupt_system_status(gasket_dev); + if (status != GASKET_STATUS_ALIVE) { + gasket_log_info(gasket_dev, + "Interrupt system reported status %d.", status); + return status; + } + + for (i = 0; i < driver_desc->num_page_tables; ++i) { + status = gasket_page_table_system_status( + gasket_dev->page_table[i]); + if (status != GASKET_STATUS_ALIVE) { + gasket_log_info( + gasket_dev, "Page table %d reported status %d.", + i, status); + return status; + } + } + + return GASKET_STATUS_ALIVE; +} + +/* + * Gasket ioctl dispatch function. + * @filp: File structure pointer describing this node usage session. + * @cmd: ioctl number to handle. + * @arg: ioctl-specific data pointer. + * + * First, checks if the ioctl is a generic ioctl. If not, it passes + * the ioctl to the ioctl_handler_cb registered in the driver description. + * If the ioctl is a generic ioctl, the function passes it to the + * gasket_ioctl_handler in gasket_ioctl.c. + */ +static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) +{ + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc; + char path[256]; + + if (!filp) + return -ENODEV; + + gasket_dev = (struct gasket_dev *)filp->private_data; + if (!gasket_dev) { + gasket_nodev_error( + "Unable to find Gasket structure for file %s", + d_path(&filp->f_path, path, 256)); + return -ENODEV; + } + + driver_desc = gasket_dev->internal_desc->driver_desc; + if (!driver_desc) { + gasket_log_error( + gasket_dev, + "Unable to find device descriptor for file %s", + d_path(&filp->f_path, path, 256)); + return -ENODEV; + } + + if (!gasket_is_supported_ioctl(cmd)) { + /* + * The ioctl handler is not a standard Gasket callback, since + * it requires different arguments. This means we can't use + * check_and_invoke_callback. + */ + if (driver_desc->ioctl_handler_cb) + return driver_desc->ioctl_handler_cb(filp, cmd, arg); + + gasket_log_error( + gasket_dev, "Received unknown ioctl 0x%x", cmd); + return -EINVAL; + } + + return gasket_handle_ioctl(filp, cmd, arg); +} + +int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type) +{ + int ret; + + mutex_lock(&gasket_dev->mutex); + ret = gasket_reset_nolock(gasket_dev, reset_type); + mutex_unlock(&gasket_dev->mutex); + return ret; +} +EXPORT_SYMBOL(gasket_reset); + +int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) +{ + int ret; + int i; + const struct gasket_driver_desc *driver_desc; + + driver_desc = gasket_dev->internal_desc->driver_desc; + if (!driver_desc->device_reset_cb) { + gasket_log_error( + gasket_dev, "No device reset callback was registered."); + return -EINVAL; + } + + /* Perform a device reset of the requested type. */ + ret = driver_desc->device_reset_cb(gasket_dev, reset_type); + if (ret) + gasket_log_error( + gasket_dev, "Device reset cb returned %d.", ret); + + /* Reinitialize the page tables and interrupt framework. */ + for (i = 0; i < driver_desc->num_page_tables; ++i) + gasket_page_table_reset(gasket_dev->page_table[i]); + + ret = gasket_interrupt_reinit(gasket_dev); + if (ret) { + gasket_log_error( + gasket_dev, "Unable to reinit interrupts: %d.", ret); + return ret; + } + + /* Get current device health. */ + gasket_dev->status = gasket_get_hw_status(gasket_dev); + if (gasket_dev->status == GASKET_STATUS_DEAD) { + gasket_log_error(gasket_dev, "Device reported as dead."); + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL(gasket_reset_nolock); + +gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( + struct gasket_dev *gasket_dev) { + return gasket_dev->internal_desc->driver_desc->ioctl_permissions_cb; +} +EXPORT_SYMBOL(gasket_get_ioctl_permissions_cb); + +static ssize_t gasket_write_mappable_regions( + char *buf, const struct gasket_driver_desc *driver_desc, int bar_index) +{ + int i; + ssize_t written; + ssize_t total_written = 0; + ulong min_addr, max_addr; + struct gasket_bar_desc bar_desc = + driver_desc->bar_descriptions[bar_index]; + + if (bar_desc.permissions == GASKET_NOMAP) + return 0; + for (i = 0; + (i < bar_desc.num_mappable_regions) && (total_written < PAGE_SIZE); + i++) { + min_addr = bar_desc.mappable_regions[i].start - + driver_desc->legacy_mmap_address_offset; + max_addr = bar_desc.mappable_regions[i].start - + driver_desc->legacy_mmap_address_offset + + bar_desc.mappable_regions[i].length_bytes; + written = scnprintf(buf, PAGE_SIZE - total_written, + "0x%08lx-0x%08lx\n", min_addr, max_addr); + total_written += written; + buf += written; + } + return total_written; +} + +static ssize_t gasket_sysfs_data_show( + struct device *device, struct device_attribute *attr, char *buf) +{ + int i, ret = 0; + ssize_t current_written = 0; + const struct gasket_driver_desc *driver_desc; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + const struct gasket_bar_desc *bar_desc; + enum gasket_sysfs_attribute_type sysfs_type; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + gasket_nodev_error( + "No sysfs mapping found for device 0x%p", device); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + gasket_nodev_error( + "No sysfs attr found for device 0x%p", device); + gasket_sysfs_put_device_data(device, gasket_dev); + return 0; + } + + driver_desc = gasket_dev->internal_desc->driver_desc; + + sysfs_type = + (enum gasket_sysfs_attribute_type)gasket_attr->data.attr_type; + switch (sysfs_type) { + case ATTR_BAR_OFFSETS: + for (i = 0; i < GASKET_NUM_BARS; i++) { + bar_desc = &driver_desc->bar_descriptions[i]; + if (bar_desc->size == 0) + continue; + current_written = + snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i, + (ulong)bar_desc->base); + buf += current_written; + ret += current_written; + } + break; + case ATTR_BAR_SIZES: + for (i = 0; i < GASKET_NUM_BARS; i++) { + bar_desc = &driver_desc->bar_descriptions[i]; + if (bar_desc->size == 0) + continue; + current_written = + snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i, + (ulong)bar_desc->size); + buf += current_written; + ret += current_written; + } + break; + case ATTR_DRIVER_VERSION: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", + gasket_dev->internal_desc->driver_desc->driver_version); + break; + case ATTR_FRAMEWORK_VERSION: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", GASKET_FRAMEWORK_VERSION); + break; + case ATTR_DEVICE_TYPE: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", + gasket_dev->internal_desc->driver_desc->name); + break; + case ATTR_HARDWARE_REVISION: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", gasket_dev->hardware_revision); + break; + case ATTR_PCI_ADDRESS: + ret = snprintf(buf, PAGE_SIZE, "%s\n", gasket_dev->kobj_name); + break; + case ATTR_STATUS: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", + gasket_num_name_lookup( + gasket_dev->status, gasket_status_name_table)); + break; + case ATTR_IS_DEVICE_OWNED: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.is_owned); + break; + case ATTR_DEVICE_OWNER: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.owner); + break; + case ATTR_WRITE_OPEN_COUNT: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.write_open_count); + break; + case ATTR_RESET_COUNT: + ret = snprintf(buf, PAGE_SIZE, "%d\n", gasket_dev->reset_count); + break; + case ATTR_USER_MEM_RANGES: + for (i = 0; i < GASKET_NUM_BARS; ++i) { + current_written = gasket_write_mappable_regions( + buf, driver_desc, i); + buf += current_written; + ret += current_written; + } + break; + default: + gasket_log_error( + gasket_dev, "Unknown attribute: %s", attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + +/* Get the driver structure for a given gasket_dev. + * @dev: pointer to gasket_dev, implementing the requested driver. + */ +const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev) +{ + return dev->internal_desc->driver_desc; +} + +/* Get the device structure for a given gasket_dev. + * @dev: pointer to gasket_dev, implementing the requested driver. + */ +struct device *gasket_get_device(struct gasket_dev *dev) +{ + if (dev->pci_dev) + return &dev->pci_dev->dev; + return NULL; +} + +/** + * Synchronously waits on device. + * @gasket_dev: Device struct. + * @bar: Bar + * @offset: Register offset + * @mask: Register mask + * @val: Expected value + * @timeout_ns: Timeout in nanoseconds + * + * Description: Busy waits for a specific combination of bits to be set + * on a Gasket register. + **/ +int gasket_wait_sync( + struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, + u64 timeout_ns) +{ + u64 reg; + struct timespec start_time, cur_time; + u64 diff_nanosec; + int count = 0; + + reg = gasket_dev_read_64(gasket_dev, bar, offset); + start_time = current_kernel_time(); + while ((reg & mask) != val) { + count++; + cur_time = current_kernel_time(); + diff_nanosec = (u64)(cur_time.tv_sec - start_time.tv_sec) * + 1000000000LL + + (u64)(cur_time.tv_nsec) - + (u64)(start_time.tv_nsec); + if (diff_nanosec > timeout_ns) { + gasket_log_error( + gasket_dev, + "gasket_wait_sync timeout: reg %llx count %x " + "dma %lld ns\n", + offset, count, diff_nanosec); + return -1; + } + reg = gasket_dev_read_64(gasket_dev, bar, offset); + } + return 0; +} +EXPORT_SYMBOL(gasket_wait_sync); + +/** + * Asynchronously waits on device. + * @gasket_dev: Device struct. + * @bar: Bar + * @offset: Register offset + * @mask: Register mask + * @val: Expected value + * @max_retries: number of sleep periods + * @delay_ms: Timeout in milliseconds + * + * Description: Busy waits for a specific combination of bits to be set on a + * Gasket register. + **/ +int gasket_wait_with_reschedule( + struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, + u64 max_retries, u64 delay_ms) +{ + u64 retries = 0; + u64 tmp; + + while (retries < max_retries) { + tmp = gasket_dev_read_64(gasket_dev, bar, offset); + if ((tmp & mask) == val) + break; + schedule_timeout(msecs_to_jiffies(delay_ms)); + retries++; + } + if (retries == max_retries) { + gasket_log_error( + gasket_dev, + "gasket_wait_with_reschedule timeout: reg %llx timeout (%llu ms)", + offset, max_retries * delay_ms); + return -EINVAL; + } + return 0; +} +EXPORT_SYMBOL(gasket_wait_with_reschedule); diff --git a/gasket_core.h b/gasket_core.h new file mode 100644 index 0000000..203b9a3 --- /dev/null +++ b/gasket_core.h @@ -0,0 +1,719 @@ +/* Gasket generic driver. Defines the set of data types and functions necessary + * to define a driver using the Gasket generic driver framework. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __GASKET_CORE_H__ +#define __GASKET_CORE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gasket_constants.h" + +/** + * struct gasket_num_name - Map numbers to names. + * @ein_num: Number. + * @ein_name: Name associated with the number, a char pointer. + * + * This structure maps numbers to names. It is used to provide printable enum + * names, e.g {0, "DEAD"} or {1, "ALIVE"}. + */ +struct gasket_num_name { + uint snn_num; + const char *snn_name; +}; + +/* + * Register location for packed interrupts. + * Each value indicates the location of an interrupt field (in units of + * gasket_driver_desc->interrupt_pack_width) within the containing register. + * In other words, this indicates the shift to use when creating a mask to + * extract/set bits within a register for a given interrupt. + */ +enum gasket_interrupt_packing { + PACK_0 = 0, + PACK_1 = 1, + PACK_2 = 2, + PACK_3 = 3, + UNPACKED = 4, +}; + +/* Type of the interrupt supported by the device. */ +enum gasket_interrupt_type { + PCI_MSIX = 0, + PCI_MSI = 1, + PLATFORM_WIRE = 2, +}; + +/* Used to describe a Gasket interrupt. Contains an interrupt index, a register, + * and packing data for that interrupt. The register and packing data + * fields is relevant only for PCI_MSIX interrupt type and can be + * set to 0 for everything else. + */ +struct gasket_interrupt_desc { + /* Device-wide interrupt index/number. */ + int index; + /* The register offset controlling this interrupt. */ + u64 reg; + /* The location of this interrupt inside register reg, if packed. */ + int packing; +}; + +/* Offsets to the wire interrupt handling registers */ +struct gasket_wire_interrupt_offsets { + u64 pending_bit_array; + u64 mask_array; +}; + +/* + * This enum is used to identify memory regions being part of the physical + * memory that belongs to a device. + */ +enum mappable_area_type { + PCI_BAR = 0, /* Default */ + BUS_REGION, /* For SYSBUS devices, i.e. AXI etc... */ + COHERENT_MEMORY +}; + +/* + * Metadata for each BAR mapping. + * This struct is used so as to track PCI memory, I/O space, AXI and coherent + * memory area... i.e. memory objects which can be referenced in the device's + * mmap function. + */ +struct gasket_bar_data { + /* Virtual base address. */ + u8 __iomem *virt_base; + + /* Physical base address. */ + ulong phys_base; + + /* Length of the mapping. */ + ulong length_bytes; + + /* Type of mappable area */ + enum mappable_area_type type; +}; + +/* Maintains device open ownership data. */ +struct gasket_ownership { + /* 1 if the device is owned, 0 otherwise. */ + int is_owned; + + /* TGID of the owner. */ + pid_t owner; + + /* Count of current device opens in write mode. */ + int write_open_count; +}; + +/* Page table modes of operation. */ +enum gasket_page_table_mode { + /* The page table is partitionable as normal, all simple by default. */ + GASKET_PAGE_TABLE_MODE_NORMAL, + + /* All entries are always simple. */ + GASKET_PAGE_TABLE_MODE_SIMPLE, + + /* All entries are always extended. No extended bit is used. */ + GASKET_PAGE_TABLE_MODE_EXTENDED, +}; + +/* Page table configuration. One per table. */ +struct gasket_page_table_config { + /* The identifier/index of this page table. */ + int id; + + /* The operation mode of this page table. */ + enum gasket_page_table_mode mode; + + /* Total (first-level) entries in this page table. */ + ulong total_entries; + + /* Base register for the page table. */ + int base_reg; + + /* + * Register containing the extended page table. This value is unused in + * GASKET_PAGE_TABLE_MODE_SIMPLE and GASKET_PAGE_TABLE_MODE_EXTENDED + * modes. + */ + int extended_reg; + + /* The bit index indicating whether a PT entry is extended. */ + int extended_bit; +}; + +/* Maintains information about a device node. */ +struct gasket_cdev_info { + /* The internal name of this device. */ + char name[GASKET_NAME_MAX]; + + /* Device number. */ + dev_t devt; + + /* Kernel-internal device structure. */ + struct device *device; + + /* Character device for real. */ + struct cdev cdev; + + /* Flag indicating if cdev_add has been called for the devices. */ + int cdev_added; + + /* Pointer to the overall gasket_dev struct for this device. */ + struct gasket_dev *gasket_dev_ptr; + + /* Ownership data for the device in question. */ + struct gasket_ownership ownership; +}; + +/* Describes the offset and length of mmapable device BAR regions. */ +struct gasket_mappable_region { + u64 start; + u64 length_bytes; +}; + +/* Describe the offset, size, and permissions for a device bar. */ +struct gasket_bar_desc { + /* + * The size of each PCI BAR range, in bytes. If a value is 0, that BAR + * will not be mapped into kernel space at all. + * For devices with 64 bit BARs, only elements 0, 2, and 4 should be + * populated, and 1, 3, and 5 should be set to 0. + * For example, for a device mapping 1M in each of the first two 64-bit + * BARs, this field would be set as { 0x100000, 0, 0x100000, 0, 0, 0 } + * (one number per bar_desc struct.) + */ + u64 size; + /* The permissions for this bar. (Should be VM_WRITE/VM_READ/VM_EXEC, + * and can be or'd.) If set to GASKET_NOMAP, the bar will + * not be used for mmapping. + */ + ulong permissions; + /* The memory address corresponding to the base of this bar, if used. */ + u64 base; + /* The number of mappable regions in this bar. */ + int num_mappable_regions; + + /* The mappable subregions of this bar. */ + const struct gasket_mappable_region *mappable_regions; + + /* Type of mappable area */ + enum mappable_area_type type; +}; + +/* Describes the offset, size, and permissions for a coherent buffer. */ +struct gasket_coherent_buffer_desc { + /* The size of the coherent buffer. */ + u64 size; + + /* The permissions for this bar. (Should be VM_WRITE/VM_READ/VM_EXEC, + * and can be or'd.) If set to GASKET_NOMAP, the bar will + * not be used for mmaping. + */ + ulong permissions; + + /* device side address. */ + u64 base; +}; + +/* Coherent buffer structure. */ +struct gasket_coherent_buffer { + /* Virtual base address. */ + u8 __iomem *virt_base; + + /* Physical base address. */ + ulong phys_base; + + /* Length of the mapping. */ + ulong length_bytes; +}; + +/* Description of Gasket-specific permissions in the mmap field. */ +enum gasket_mapping_options { GASKET_NOMAP = 0 }; + +/* This struct represents an undefined bar that should never be mapped. */ +#define GASKET_UNUSED_BAR \ + { \ + 0, GASKET_NOMAP, 0, 0, NULL, 0 \ + } + +/* Internal data for a Gasket device. See gasket_core.c for more information. */ +struct gasket_internal_desc; + +#define MAX_NUM_COHERENT_PAGES 16 + +/* + * Device data for Gasket device instances. + * + * This structure contains the data required to manage a Gasket device. + */ +struct gasket_dev { + /* Pointer to the internal driver description for this device. */ + struct gasket_internal_desc *internal_desc; + + /* PCI subsystem metadata. */ + struct pci_dev *pci_dev; + + /* This device's index into internal_desc->devs. */ + int dev_idx; + + /* The name of this device, as reported by the kernel. */ + char kobj_name[GASKET_NAME_MAX]; + + /* Virtual address of mapped BAR memory range. */ + struct gasket_bar_data bar_data[GASKET_NUM_BARS]; + + /* Coherent buffer. */ + struct gasket_coherent_buffer coherent_buffer; + + /* Number of page tables for this device. */ + int num_page_tables; + + /* Address translations. Page tables have a private implementation. */ + struct gasket_page_table *page_table[GASKET_MAX_NUM_PAGE_TABLES]; + + /* Interrupt data for this device. */ + struct gasket_interrupt_data *interrupt_data; + + /* Status for this device - GASKET_STATUS_ALIVE or _DEAD. */ + uint status; + + /* Number of times this device has been reset. */ + uint reset_count; + + /* Dev information for the cdev node. */ + struct gasket_cdev_info dev_info; + + /* Hardware revision value for this device. */ + int hardware_revision; + + /* + * Device-specific data; allocated in gasket_driver_desc.add_dev_cb() + * and freed in gasket_driver_desc.remove_dev_cb(). + */ + void *cb_data; + + /* Protects access to per-device data (i.e. this structure). */ + struct mutex mutex; + + /* cdev hash tracking/membership structure, Accel and legacy. */ + /* Unused until Accel is upstreamed. */ + struct hlist_node hlist_node; + struct hlist_node legacy_hlist_node; +}; + +/* Type of the ioctl permissions check callback. See below. */ +typedef int (*gasket_ioctl_permissions_cb_t)( + struct file *filp, uint cmd, ulong arg); + +/* + * Device type descriptor. + * + * This structure contains device-specific data needed to identify and address a + * type of device to be administered via the Gasket generic driver. + * + * Device IDs are per-driver. In other words, two drivers using the Gasket + * framework will each have a distinct device 0 (for example). + */ +struct gasket_driver_desc { + /* The name of this device type. */ + const char *name; + + /* The name of this specific device model. */ + const char *chip_model; + + /* The version of the chip specified in chip_model. */ + const char *chip_version; + + /* The version of this driver: "1.0.0", "2.1.3", etc. */ + const char *driver_version; + + /* + * Non-zero if we should create "legacy" (device and device-class- + * specific) character devices and sysfs nodes. + */ + /* Unused until Accel is upstreamed. */ + int legacy_support; + + /* Major and minor numbers identifying the device. */ + int major, minor; + + /* Module structure for this driver. */ + struct module *module; + + /* PCI ID table. */ + const struct pci_device_id *pci_id_table; + + /* The number of page tables handled by this driver. */ + int num_page_tables; + + /* The index of the bar containing the page tables. */ + int page_table_bar_index; + + /* Registers used to control each page table. */ + const struct gasket_page_table_config *page_table_configs; + + /* The bit index indicating whether a PT entry is extended. */ + int page_table_extended_bit; + + /* + * Legacy mmap address adjusment for legacy devices only. Should be 0 + * for any new device. + */ + ulong legacy_mmap_address_offset; + + /* Set of 6 bar descriptions that describe all PCIe bars. + * Note that BUS/AXI devices (i.e. non PCI devices) use those. + */ + struct gasket_bar_desc bar_descriptions[GASKET_NUM_BARS]; + + /* + * Coherent buffer description. + */ + struct gasket_coherent_buffer_desc coherent_buffer_description; + + /* Offset of wire interrupt registers. */ + const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets; + + /* Interrupt type. (One of gasket_interrupt_type). */ + int interrupt_type; + + /* Index of the bar containing the interrupt registers to program. */ + int interrupt_bar_index; + + /* Number of interrupts in the gasket_interrupt_desc array */ + int num_interrupts; + + /* Description of the interrupts for this device. */ + const struct gasket_interrupt_desc *interrupts; + + /* + * If this device packs multiple interrupt->MSI-X mappings into a + * single register (i.e., "uses packed interrupts"), only a single bit + * width is supported for each interrupt mapping (unpacked/"full-width" + * interrupts are always supported). This value specifies that width. If + * packed interrupts are not used, this value is ignored. + */ + int interrupt_pack_width; + + /* Driver callback functions - all may be NULL */ + /* + * add_dev_cb: Callback when a device is found. + * @dev: The gasket_dev struct for this driver instance. + * + * This callback should initialize the device-specific cb_data. + * Called when a device is found by the driver, + * before any BAR ranges have been mapped. If this call fails (returns + * nonzero), remove_dev_cb will be called. + * + */ + int (*add_dev_cb)(struct gasket_dev *dev); + + /* + * remove_dev_cb: Callback for when a device is removed from the system. + * @dev: The gasket_dev struct for this driver instance. + * + * This callback should free data allocated in add_dev_cb. + * Called immediately before a device is unregistered by the driver. + * All framework-managed resources will have been cleaned up by the time + * this callback is invoked (PCI BARs, character devices, ...). + */ + int (*remove_dev_cb)(struct gasket_dev *dev); + + /* + * device_open_cb: Callback for when a device node is opened in write + * mode. + * @dev: The gasket_dev struct for this driver instance. + * + * This callback should perform device-specific setup that needs to + * occur only once when a device is first opened. + */ + int (*device_open_cb)(struct gasket_dev *dev); + + /* + * device_release_cb: Callback when a device is closed. + * @gasket_dev: The gasket_dev struct for this driver instance. + * + * This callback is called whenever a device node fd is closed, as + * opposed to device_close_cb, which is called when the _last_ + * descriptor for an open file is closed. This call is intended to + * handle any per-user or per-fd cleanup. + */ + int (*device_release_cb)( + struct gasket_dev *gasket_dev, struct file *file); + + /* + * device_close_cb: Callback for when a device node is closed for the + * last time. + * @dev: The gasket_dev struct for this driver instance. + * + * This callback should perform device-specific cleanup that only + * needs to occur when the last reference to a device node is closed. + * + * This call is intended to handle and device-wide cleanup, as opposed + * to per-fd cleanup (which should be handled by device_release_cb). + */ + int (*device_close_cb)(struct gasket_dev *dev); + + /* + * enable_dev_cb: Callback immediately before enabling the device. + * @dev: Pointer to the gasket_dev struct for this driver instance. + * + * This callback is invoked after the device has been added and all BAR + * spaces mapped, immediately before registering and enabling the + * [character] device via cdev_add. If this call fails (returns + * nonzero), disable_dev_cb will be called. + * + * Note that cdev are initialized but not active + * (cdev_add has not yet been called) when this callback is invoked. + */ + int (*enable_dev_cb)(struct gasket_dev *dev); + + /* + * disable_dev_cb: Callback immediately after disabling the device. + * @dev: Pointer to the gasket_dev struct for this driver instance. + * + * Called during device shutdown, immediately after disabling device + * operations via cdev_del. + */ + int (*disable_dev_cb)(struct gasket_dev *dev); + + /* + * sysfs_setup_cb: Callback to set up driver-specific sysfs nodes. + * @dev: Pointer to the gasket_dev struct for this device. + * + * Called just before enable_dev_cb. + * + */ + int (*sysfs_setup_cb)(struct gasket_dev *dev); + + /* + * sysfs_cleanup_cb: Callback to clean up driver-specific sysfs nodes. + * @dev: Pointer to the gasket_dev struct for this device. + * + * Called just before disable_dev_cb. + * + */ + int (*sysfs_cleanup_cb)(struct gasket_dev *dev); + + /* + * get_mappable_regions_cb: Get descriptors of mappable device memory. + * @gasket_dev: Pointer to the struct gasket_dev for this device. + * @bar_index: BAR for which to retrieve memory ranges. + * @mappable_regions: Out-pointer to the list of mappable regions on the + * device/BAR for this process. + * @num_mappable_regions: Out-pointer for the size of mappable_regions. + * + * Called when handling mmap(), this callback is used to determine which + * regions of device memory may be mapped by the current process. This + * information is then compared to mmap request to determine which + * regions to actually map. + */ + int (*get_mappable_regions_cb)( + struct gasket_dev *gasket_dev, int bar_index, + struct gasket_mappable_region **mappable_regions, + int *num_mappable_regions); + + /* + * ioctl_permissions_cb: Check permissions for generic ioctls. + * @filp: File structure pointer describing this node usage session. + * @cmd: ioctl number to handle. + * @arg: ioctl-specific data pointer. + * + * Returns 1 if the ioctl may be executed, 0 otherwise. If this callback + * isn't specified a default routine will be used, that only allows the + * original device opener (i.e, the "owner") to execute state-affecting + * ioctls. + */ + gasket_ioctl_permissions_cb_t ioctl_permissions_cb; + + /* + * ioctl_handler_cb: Callback to handle device-specific ioctls. + * @filp: File structure pointer describing this node usage session. + * @cmd: ioctl number to handle. + * @arg: ioctl-specific data pointer. + * + * Invoked whenever an ioctl is called that the generic Gasket + * framework doesn't support. If no cb is registered, unknown ioctls + * return -EINVAL. Should return an error status (either -EINVAL or + * the error result of the ioctl being handled). + */ + long (*ioctl_handler_cb)(struct file *filp, uint cmd, ulong arg); + + /* + * device_status_cb: Callback to determine device health. + * @dev: Pointer to the gasket_dev struct for this device. + * + * Called to determine if the device is healthy or not. Should return + * a member of the gasket_status_type enum. + * + */ + int (*device_status_cb)(struct gasket_dev *dev); + + /* + * hardware_revision_cb: Get the device's hardware revision. + * @dev: Pointer to the gasket_dev struct for this device. + * + * Called to determine the reported rev of the physical hardware. + * Revision should be >0. A negative return value is an error. + */ + int (*hardware_revision_cb)(struct gasket_dev *dev); + + /* + * device_reset_cb: Reset the hardware in question. + * @dev: Pointer to the gasket_dev structure for this device. + * @type: Integer representing reset type. (All + * Gasket resets have an integer representing their type + * defined in (device)_ioctl.h; the specific resets are + * device-dependent, but are handled in the device-specific + * callback anyways.) + * + * Called by reset ioctls. This function should not + * lock the gasket_dev mutex. It should return 0 on success + * and an error on failure. + */ + int (*device_reset_cb)(struct gasket_dev *dev, uint reset_type); +}; + +/* + * Register the specified device type with the framework. + * @desc: Populated/initialized device type descriptor. + * + * This function does _not_ take ownership of desc; the underlying struct must + * exist until the matching call to gasket_unregister_device. + * This function should be called from your driver's module_init function. + */ +int gasket_register_device(const struct gasket_driver_desc *desc); + +/* + * Remove the specified device type from the framework. + * @desc: Descriptor for the device type to unregister; it should have been + * passed to gasket_register_device in a previous call. + * + * This function should be called from your driver's module_exit function. + */ +void gasket_unregister_device(const struct gasket_driver_desc *desc); + +/* + * Reset the Gasket device. + * @gasket_dev: Gasket device struct. + * @reset_type: Uint representing requested reset type. Should be + * valid in the underlying callback. + * + * Calls device_reset_cb. Returns 0 on success and an error code othewrise. + * gasket_reset_nolock will not lock the mutex, gasket_reset will. + * + */ +int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type); +int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type); + +/* + * Memory management functions. These will likely be spun off into their own + * file in the future. + */ + +/* Unmaps the specified mappable region from a VMA. */ +int gasket_mm_unmap_region( + const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, + const struct gasket_mappable_region *map_region); + +/* + * Get the ioctl permissions callback. + * @gasket_dev: Gasket device structure. + */ +gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( + struct gasket_dev *gasket_dev); + +/** + * Lookup a name by number in a num_name table. + * @num: Number to lookup. + * @table: Array of num_name structures, the table for the lookup. + * + */ +const char *gasket_num_name_lookup( + uint num, const struct gasket_num_name *table); + +/* Handy inlines */ +static inline ulong gasket_dev_read_64( + struct gasket_dev *gasket_dev, int bar, ulong location) +{ + return readq(&gasket_dev->bar_data[bar].virt_base[location]); +} + +static inline void gasket_dev_write_64( + struct gasket_dev *dev, u64 value, int bar, ulong location) +{ + writeq(value, &dev->bar_data[bar].virt_base[location]); +} + +static inline void gasket_dev_write_32( + struct gasket_dev *dev, u32 value, int bar, ulong location) +{ + writel(value, &dev->bar_data[bar].virt_base[location]); +} + +static inline u32 gasket_dev_read_32( + struct gasket_dev *dev, int bar, ulong location) +{ + return readl(&dev->bar_data[bar].virt_base[location]); +} + +static inline void gasket_read_modify_write_64( + struct gasket_dev *dev, int bar, ulong location, u64 value, + u64 mask_width, u64 mask_shift) +{ + u64 mask, tmp; + + tmp = gasket_dev_read_64(dev, bar, location); + mask = ((1 << mask_width) - 1) << mask_shift; + tmp = (tmp & ~mask) | (value << mask_shift); + gasket_dev_write_64(dev, tmp, bar, location); +} + +static inline void gasket_read_modify_write_32( + struct gasket_dev *dev, int bar, ulong location, u32 value, + u32 mask_width, u32 mask_shift) +{ + u32 mask, tmp; + + tmp = gasket_dev_read_32(dev, bar, location); + mask = ((1 << mask_width) - 1) << mask_shift; + tmp = (tmp & ~mask) | (value << mask_shift); + gasket_dev_write_32(dev, tmp, bar, location); +} + +/* Get the Gasket driver structure for a given device. */ +const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev); + +/* Get the device structure for a given device. */ +struct device *gasket_get_device(struct gasket_dev *dev); + +/* Helper function, Synchronous waits on a given set of bits. */ +int gasket_wait_sync( + struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, + u64 timeout_ns); + +/* Helper function, Asynchronous waits on a given set of bits. */ +int gasket_wait_with_reschedule( + struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, + u64 max_retries, u64 delay_ms); + +#endif /* __GASKET_CORE_H__ */ diff --git a/gasket_interrupt.c b/gasket_interrupt.c new file mode 100644 index 0000000..b74eefe --- /dev/null +++ b/gasket_interrupt.c @@ -0,0 +1,638 @@ +/* Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "gasket_interrupt.h" + +#include "gasket_constants.h" +#include "gasket_core.h" +#include "gasket_logging.h" +#include "gasket_sysfs.h" +#include +#include +#ifdef GASKET_KERNEL_TRACE_SUPPORT +#define CREATE_TRACE_POINTS +#include +#else +#define trace_gasket_interrupt_event(x, ...) +#endif +/* Retry attempts if the requested number of interrupts aren't available. */ +#define MSIX_RETRY_COUNT 3 + +/* Instance interrupt management data. */ +struct gasket_interrupt_data { + /* The name associated with this interrupt data. */ + const char *name; + + /* Interrupt type. See gasket_interrupt_type in gasket_core.h */ + int type; + + /* The PCI device [if any] associated with the owning device. */ + struct pci_dev *pci_dev; + + /* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */ + int msix_configured; + + /* The number of interrupts requested by the owning device. */ + int num_interrupts; + + /* A pointer to the interrupt descriptor struct for this device. */ + const struct gasket_interrupt_desc *interrupts; + + /* The index of the bar into which interrupts should be mapped. */ + int interrupt_bar_index; + + /* The width of a single interrupt in a packed interrupt register. */ + int pack_width; + + /* offset of wire interrupt registers */ + const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets; + + /* + * Design-wise, these elements should be bundled together, but + * pci_enable_msix's interface requires that they be managed + * individually (requires array of struct msix_entry). + */ + + /* The number of successfully configured interrupts. */ + int num_configured; + + /* The MSI-X data for each requested/configured interrupt. */ + struct msix_entry *msix_entries; + + /* The eventfd "callback" data for each interrupt. */ + struct eventfd_ctx **eventfd_ctxs; + + /* The number of times each interrupt has been called. */ + ulong *interrupt_counts; + + /* Linux IRQ number. */ + int irq; +}; + +/* Function definitions. */ +static ssize_t interrupt_sysfs_show( + struct device *device, struct device_attribute *attr, char *buf); + +static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id); + +/* Structures to display interrupt counts in sysfs. */ +enum interrupt_sysfs_attribute_type { + ATTR_INTERRUPT_COUNTS, +}; + +static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = { + GASKET_SYSFS_RO( + interrupt_counts, interrupt_sysfs_show, ATTR_INTERRUPT_COUNTS), + GASKET_END_OF_ATTR_ARRAY, +}; + +/* + * Set up device registers for interrupt handling. + * @gasket_dev: The Gasket information structure for this device. + * + * Sets up the device registers with the correct indices for the relevant + * interrupts. + */ +static void gasket_interrupt_setup(struct gasket_dev *gasket_dev); + +/* MSIX init and cleanup. */ +static int gasket_interrupt_msix_init( + struct gasket_interrupt_data *interrupt_data); +static void gasket_interrupt_msix_cleanup( + struct gasket_interrupt_data *interrupt_data); +static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev); + +int gasket_interrupt_init( + struct gasket_dev *gasket_dev, const char *name, int type, + const struct gasket_interrupt_desc *interrupts, + int num_interrupts, int pack_width, int bar_index, + const struct gasket_wire_interrupt_offsets *wire_int_offsets) +{ + int ret; + struct gasket_interrupt_data *interrupt_data; + + interrupt_data = kzalloc( + sizeof(struct gasket_interrupt_data), GFP_KERNEL); + if (!interrupt_data) + return -ENOMEM; + gasket_dev->interrupt_data = interrupt_data; + interrupt_data->name = name; + interrupt_data->type = type; + interrupt_data->pci_dev = gasket_dev->pci_dev; + interrupt_data->num_interrupts = num_interrupts; + interrupt_data->interrupts = interrupts; + interrupt_data->interrupt_bar_index = bar_index; + interrupt_data->pack_width = pack_width; + interrupt_data->num_configured = 0; + interrupt_data->wire_interrupt_offsets = wire_int_offsets; + + /* Allocate all dynamic structures. */ + interrupt_data->msix_entries = kzalloc( + sizeof(struct msix_entry) * num_interrupts, GFP_KERNEL); + if (!interrupt_data->msix_entries) { + kfree(interrupt_data); + return -ENOMEM; + } + + interrupt_data->eventfd_ctxs = kzalloc( + sizeof(struct eventfd_ctx *) * num_interrupts, GFP_KERNEL); + if (!interrupt_data->eventfd_ctxs) { + kfree(interrupt_data->msix_entries); + kfree(interrupt_data); + return -ENOMEM; + } + + interrupt_data->interrupt_counts = kzalloc( + sizeof(ulong) * num_interrupts, GFP_KERNEL); + if (!interrupt_data->interrupt_counts) { + kfree(interrupt_data->eventfd_ctxs); + kfree(interrupt_data->msix_entries); + kfree(interrupt_data); + return -ENOMEM; + } + + switch (interrupt_data->type) { + case PCI_MSIX: + ret = gasket_interrupt_msix_init(interrupt_data); + if (ret) + break; + force_msix_interrupt_unmasking(gasket_dev); + break; + + case PCI_MSI: + case PLATFORM_WIRE: + default: + gasket_nodev_error( + "Cannot handle unsupported interrupt type %d.", + interrupt_data->type); + ret = -EINVAL; + }; + + if (ret) { + /* Failing to setup interrupts will cause the device to report + * GASKET_STATUS_LAMED. But it is not fatal. + */ + gasket_log_warn( + gasket_dev, "Couldn't initialize interrupts: %d", ret); + return 0; + } + + gasket_interrupt_setup(gasket_dev); + gasket_sysfs_create_entries( + gasket_dev->dev_info.device, interrupt_sysfs_attrs); + + return 0; +} + +static int gasket_interrupt_msix_init( + struct gasket_interrupt_data *interrupt_data) +{ + int ret = 1; + int i; + + for (i = 0; i < interrupt_data->num_interrupts; i++) { + interrupt_data->msix_entries[i].entry = i; + interrupt_data->msix_entries[i].vector = 0; + interrupt_data->eventfd_ctxs[i] = NULL; + } + + /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */ + for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++) + ret = pci_enable_msix_exact(interrupt_data->pci_dev, + interrupt_data->msix_entries, + interrupt_data->num_interrupts); + + if (ret) + return ret > 0 ? -EBUSY : ret; + interrupt_data->msix_configured = 1; + + for (i = 0; i < interrupt_data->num_interrupts; i++) { + ret = request_irq( + interrupt_data->msix_entries[i].vector, + gasket_msix_interrupt_handler, 0, interrupt_data->name, + interrupt_data); + + if (ret) { + gasket_nodev_error( + "Cannot get IRQ for interrupt %d, vector %d; " + "%d\n", + i, interrupt_data->msix_entries[i].vector, ret); + return ret; + } + + interrupt_data->num_configured++; + } + + return 0; +} + +static void gasket_interrupt_msix_cleanup( + struct gasket_interrupt_data *interrupt_data) +{ + int i; + + for (i = 0; i < interrupt_data->num_configured; i++) + free_irq(interrupt_data->msix_entries[i].vector, + interrupt_data); + interrupt_data->num_configured = 0; + + if (interrupt_data->msix_configured) + pci_disable_msix(interrupt_data->pci_dev); + interrupt_data->msix_configured = 0; +} + +/* + * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt + * setup code with MSIX vectors masked. This is wrong because nothing else in + * the driver will normally touch the MSIX vectors. + * + * As a temporary hack, force unmasking there. + * + * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after + * gasket_interrupt_msix_init(), and remove this code. + */ +static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev) +{ + int i; +#define MSIX_VECTOR_SIZE 16 +#define MSIX_MASK_BIT_OFFSET 12 +#define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800 + for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) { + /* Check if the MSIX vector is unmasked */ + ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE + + MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE; + u32 mask = + gasket_dev_read_32( + gasket_dev, + gasket_dev->interrupt_data->interrupt_bar_index, + location); + if (!(mask & 1)) + continue; + /* Unmask the msix vector (clear 32 bits) */ + gasket_dev_write_32( + gasket_dev, 0, + gasket_dev->interrupt_data->interrupt_bar_index, + location); + } +#undef MSIX_VECTOR_SIZE +#undef MSIX_MASK_BIT_OFFSET +#undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +} + +int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) +{ + int ret; + + if (!gasket_dev->interrupt_data) { + gasket_log_error( + gasket_dev, + "Attempted to reinit uninitialized interrupt data."); + return -EINVAL; + } + + switch (gasket_dev->interrupt_data->type) { + case PCI_MSIX: + gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data); + ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data); + if (ret) + break; + force_msix_interrupt_unmasking(gasket_dev); + break; + + case PCI_MSI: + case PLATFORM_WIRE: + default: + gasket_nodev_error( + "Cannot handle unsupported interrupt type %d.", + gasket_dev->interrupt_data->type); + ret = -EINVAL; + }; + + if (ret) { + /* Failing to setup MSIx will cause the device + * to report GASKET_STATUS_LAMED, but is not fatal. + */ + gasket_log_warn(gasket_dev, "Couldn't init msix: %d", ret); + return 0; + } + + gasket_interrupt_setup(gasket_dev); + + return 0; +} + +/* See gasket_interrupt.h for description. */ +int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev) +{ + gasket_log_debug(gasket_dev, "Clearing interrupt counts."); + memset(gasket_dev->interrupt_data->interrupt_counts, 0, + gasket_dev->interrupt_data->num_interrupts * + sizeof(*gasket_dev->interrupt_data->interrupt_counts)); + return 0; +} + +/* + * Set up device registers for interrupt handling. + * @gasket_dev: The Gasket information structure for this device. + * + * Sets up the device registers with the correct indices for the relevant + * interrupts. + */ +static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) +{ + int i; + int pack_shift; + ulong mask; + ulong value; + struct gasket_interrupt_data *interrupt_data = + gasket_dev->interrupt_data; + + if (!interrupt_data) { + gasket_log_error( + gasket_dev, "Interrupt data is not initialized."); + return; + } + + gasket_log_debug(gasket_dev, "Running interrupt setup."); + + if (interrupt_data->type == PLATFORM_WIRE || + interrupt_data->type == PCI_MSI) { + /* Nothing needs to be done for platform or PCI devices. */ + return; + } + + if (interrupt_data->type != PCI_MSIX) { + gasket_nodev_error( + "Cannot handle unsupported interrupt type %d.", + interrupt_data->type); + return; + } + + /* Setup the MSIX table. */ + + for (i = 0; i < interrupt_data->num_interrupts; i++) { + /* + * If the interrupt is not packed, we can write the index into + * the register directly. If not, we need to deal with a read- + * modify-write and shift based on the packing index. + */ + gasket_log_debug( + gasket_dev, + "Setting up interrupt index %d with index 0x%llx and " + "packing %d", + interrupt_data->interrupts[i].index, + interrupt_data->interrupts[i].reg, + interrupt_data->interrupts[i].packing); + if (interrupt_data->interrupts[i].packing == UNPACKED) { + value = interrupt_data->interrupts[i].index; + } else { + switch (interrupt_data->interrupts[i].packing) { + case PACK_0: + pack_shift = 0; + break; + case PACK_1: + pack_shift = interrupt_data->pack_width; + break; + case PACK_2: + pack_shift = 2 * interrupt_data->pack_width; + break; + case PACK_3: + pack_shift = 3 * interrupt_data->pack_width; + break; + default: + gasket_nodev_error( + "Found interrupt description with " + "unknown enum %d.", + interrupt_data->interrupts[i].packing); + return; + } + + mask = ~(0xFFFF << pack_shift); + value = gasket_dev_read_64( + gasket_dev, + interrupt_data->interrupt_bar_index, + interrupt_data->interrupts[i].reg) & + mask; + value |= interrupt_data->interrupts[i].index + << pack_shift; + } + gasket_dev_write_64(gasket_dev, value, + interrupt_data->interrupt_bar_index, + interrupt_data->interrupts[i].reg); + } +} + +/* See gasket_interrupt.h for description. */ +void gasket_interrupt_pause(struct gasket_dev *gasket_dev, int enable_pause) +{ + WARN_ON(!gasket_dev); + + if (!gasket_dev->interrupt_data) + return; /* nothing to do */ + + if (gasket_dev->interrupt_data->type == PCI_MSI || + gasket_dev->interrupt_data->type == PCI_MSIX) { + /* Nothing to be done for MSI/MSIX just yet. */ + } + + if (gasket_dev->interrupt_data->type == PLATFORM_WIRE) { + /* Nothing to be done for PLATFORM_WIRE */ + } +} +EXPORT_SYMBOL(gasket_interrupt_pause); + +void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) +{ + struct gasket_interrupt_data *interrupt_data = + gasket_dev->interrupt_data; + /* + * It is possible to get an error code from gasket_interrupt_init + * before interrupt_data has been allocated, so check it. + */ + if (!interrupt_data) + return; + + switch (interrupt_data->type) { + case PCI_MSIX: + gasket_interrupt_msix_cleanup(interrupt_data); + break; + + case PCI_MSI: + case PLATFORM_WIRE: + default: + gasket_nodev_error( + "Cannot handle unsupported interrupt type %d.", + interrupt_data->type); + }; + + kfree(interrupt_data->interrupt_counts); + kfree(interrupt_data->eventfd_ctxs); + kfree(interrupt_data->msix_entries); + kfree(interrupt_data); + gasket_dev->interrupt_data = NULL; +} + +int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) +{ + if (!gasket_dev->interrupt_data) { + gasket_nodev_info("Interrupt data is null."); + return GASKET_STATUS_DEAD; + } + + if (!gasket_dev->interrupt_data->msix_configured) { + gasket_nodev_info("Interrupt not initialized."); + return GASKET_STATUS_LAMED; + } + + if (gasket_dev->interrupt_data->num_configured != + gasket_dev->interrupt_data->num_interrupts) { + gasket_nodev_info("Not all interrupts were configured."); + return GASKET_STATUS_LAMED; + } + + return GASKET_STATUS_ALIVE; +} + +int gasket_interrupt_set_eventfd( + struct gasket_interrupt_data *interrupt_data, int interrupt, + int event_fd) +{ + struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd); + + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + if (interrupt < 0 || interrupt > interrupt_data->num_interrupts) + return -EINVAL; + + interrupt_data->eventfd_ctxs[interrupt] = ctx; + return 0; +} + +int gasket_interrupt_clear_eventfd( + struct gasket_interrupt_data *interrupt_data, int interrupt) +{ + if (interrupt < 0 || interrupt > interrupt_data->num_interrupts) + return -EINVAL; + + interrupt_data->eventfd_ctxs[interrupt] = NULL; + return 0; +} + +int gasket_interrupt_trigger_eventfd( + struct gasket_interrupt_data *interrupt_data, int interrupt) +{ + struct eventfd_ctx *ctx = interrupt_data->eventfd_ctxs[interrupt]; + + if (!ctx) + return -EINVAL; + + eventfd_signal(ctx, 1); + return 0; +} + +struct msix_entry *gasket_interrupt_get_msix_entries( + struct gasket_interrupt_data *interrupt_data) +{ + return interrupt_data->msix_entries; +} + +struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs( + struct gasket_interrupt_data *interrupt_data) +{ + return interrupt_data->eventfd_ctxs; +} +EXPORT_SYMBOL(gasket_interrupt_get_eventfd_ctxs); + +static ssize_t interrupt_sysfs_show( + struct device *device, struct device_attribute *attr, char *buf) +{ + int i, ret; + ssize_t written = 0, total_written = 0; + struct gasket_interrupt_data *interrupt_data; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + enum interrupt_sysfs_attribute_type sysfs_type; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + gasket_nodev_error( + "No sysfs mapping found for device 0x%p", device); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + gasket_nodev_error( + "No sysfs attr data found for device 0x%p", device); + gasket_sysfs_put_device_data(device, gasket_dev); + return 0; + } + + sysfs_type = (enum interrupt_sysfs_attribute_type) + gasket_attr->data.attr_type; + interrupt_data = gasket_dev->interrupt_data; + switch (sysfs_type) { + case ATTR_INTERRUPT_COUNTS: + for (i = 0; i < interrupt_data->num_interrupts; ++i) { + written = + scnprintf(buf, PAGE_SIZE - total_written, + "0x%02x: %ld\n", i, + interrupt_data->interrupt_counts[i]); + total_written += written; + buf += written; + } + ret = total_written; + break; + default: + gasket_log_error( + gasket_dev, "Unknown attribute: %s", attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + +/* + * MSIX interrupt handler, used with PCI driver. + */ +static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) +{ + struct eventfd_ctx *ctx; + struct gasket_interrupt_data *interrupt_data = dev_id; + int interrupt = -1; + int i; + + /* If this linear lookup is a problem, we can maintain a map/hash. */ + for (i = 0; i < interrupt_data->num_interrupts; i++) { + if (interrupt_data->msix_entries[i].vector == irq) { + interrupt = interrupt_data->msix_entries[i].entry; + break; + } + } + if (interrupt == -1) { + gasket_nodev_error("Received unknown irq %d", irq); + return IRQ_HANDLED; + } + trace_gasket_interrupt_event(interrupt_data->name, interrupt); + + ctx = interrupt_data->eventfd_ctxs[interrupt]; + if (ctx) + eventfd_signal(ctx, 1); + + ++(interrupt_data->interrupt_counts[interrupt]); + + return IRQ_HANDLED; +} diff --git a/gasket_interrupt.h b/gasket_interrupt.h new file mode 100644 index 0000000..3a8afae --- /dev/null +++ b/gasket_interrupt.h @@ -0,0 +1,172 @@ +/* + * Gasket common interrupt module. Defines functions for enabling + * eventfd-triggered interrupts between a Gasket device and a host process. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __GASKET_INTERRUPT_H__ +#define __GASKET_INTERRUPT_H__ + +#include +#include + +#include "gasket_core.h" + +/* Note that this currently assumes that device interrupts are a dense set, + * numbered from 0 - (num_interrupts - 1). Should this have to change, these + * APIs will have to be updated. + */ + +/* Opaque type used to hold interrupt subsystem data. */ +struct gasket_interrupt_data; + +/* + * Initialize the interrupt module. + * @gasket_dev: The Gasket device structure for the device to be initted. + * @type: Type of the interrupt. (See gasket_interrupt_type). + * @name: The name to associate with these interrupts. + * @interrupts: An array of all interrupt descriptions for this device. + * @num_interrupts: The length of the @interrupts array. + * @pack_width: The width, in bits, of a single field in a packed interrupt reg. + * @bar_index: The bar containing all interrupt registers. + * + * Allocates and initializes data to track interrupt state for a device. + * After this call, no interrupts will be configured/delivered; call + * gasket_interrupt_set_vector[_packed] to associate each interrupt with an + * __iomem location, then gasket_interrupt_set_eventfd to associate an eventfd + * with an interrupt. + * + * If num_interrupts interrupts are not available, this call will return a + * negative error code. In that case, gasket_interrupt_cleanup should still be + * called. Returns 0 on success (which can include a device where interrupts + * are not possible to set up, but is otherwise OK; that device will report + * status LAMED.) + */ +int gasket_interrupt_init( + struct gasket_dev *gasket_dev, const char *name, int type, + const struct gasket_interrupt_desc *interrupts, + int num_interrupts, int pack_width, int bar_index, + const struct gasket_wire_interrupt_offsets *wire_int_offsets); + +/* + * Clean up a device's interrupt structure. + * @gasket_dev: The Gasket information structure for this device. + * + * Cleans up the device's interrupts and deallocates data. + */ +void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev); + +/* + * Clean up and re-initialize the MSI-x subsystem. + * @gasket_dev: The Gasket information structure for this device. + * + * Performs a teardown of the MSI-x subsystem and re-initializes it. Does not + * free the underlying data structures. Returns 0 on success and an error code + * on error. + */ +int gasket_interrupt_reinit(struct gasket_dev *gasket_dev); + +/* + * Reset the counts stored in the interrupt subsystem. + * @gasket_dev: The Gasket information structure for this device. + * + * Sets the counts of all interrupts in the subsystem to 0. + */ +int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev); + +/* + * Associates an eventfd with a device interrupt. + * @data: Pointer to device interrupt data. + * @interrupt: The device interrupt to configure. + * @event_fd: The eventfd to associate with the interrupt. + * + * Prepares the host to receive notification of device interrupts by associating + * event_fd with interrupt. Upon receipt of a device interrupt, event_fd will be + * signaled, after successful configuration. + * + * Returns 0 on success, a negative error code otherwise. + */ +int gasket_interrupt_set_eventfd( + struct gasket_interrupt_data *interrupt_data, int interrupt, + int event_fd); + +/* + * Removes an interrupt-eventfd association. + * @data: Pointer to device interrupt data. + * @interrupt: The device interrupt to de-associate. + * + * Removes any eventfd associated with the specified interrupt, if any. + */ +int gasket_interrupt_clear_eventfd( + struct gasket_interrupt_data *interrupt_data, int interrupt); + +/* + * Signals the eventfd associated with interrupt. + * @data: Pointer to device interrupt data. + * @interrupt: The device interrupt to signal for. + * + * Simulates a device interrupt by signaling the eventfd associated with + * interrupt, if any. + * Returns 0 if the eventfd was successfully triggered, a negative error code + * otherwise (if, for example, no eventfd was associated with interrupt). + */ +int gasket_interrupt_trigger_eventfd( + struct gasket_interrupt_data *interrupt_data, int interrupt); + +/* + * The below functions exist for backwards compatibility. + * No new uses should be written. + */ +/* + * Retrieve a pointer to data's MSI-X + * entries. + * @data: The interrupt data from which to extract. + * + * Returns the internal pointer to data's MSI-X entries. + */ +struct msix_entry *gasket_interrupt_get_msix_entries( + struct gasket_interrupt_data *interrupt_data); + +/* + * Get a pointer to data's eventfd contexts. + * @data: The interrupt data from which to extract. + * + * Returns the internal pointer to data's eventfd contexts. + * + * This function exists for backwards compatibility with older drivers. + * No new uses should be written. + */ +struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs( + struct gasket_interrupt_data *interrupt_data); + +/* + * Get the health of the interrupt subsystem. + * @gasket_dev: The Gasket device struct. + * + * Returns DEAD if not set up, LAMED if initialization failed, and ALIVE + * otherwise. + */ + +int gasket_interrupt_system_status(struct gasket_dev *gasket_dev); + +/* + * Masks interrupts and de-register the handler. + * After an interrupt pause it is not guaranteed that the chip registers will + * be accessible anymore, since the chip may be in a power save mode, + * which means that the interrupt handler (if it were to happen) may not + * have a way to clear the interrupt condition. + * @gasket_dev: The Gasket device struct + * @enable_pause: Whether to pause or unpause the interrupts. + */ +void gasket_interrupt_pause(struct gasket_dev *gasket_dev, int enable_pause); + +#endif diff --git a/gasket_ioctl.c b/gasket_ioctl.c new file mode 100644 index 0000000..4758083 --- /dev/null +++ b/gasket_ioctl.c @@ -0,0 +1,449 @@ +/* Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include "gasket.h" +#include "gasket_ioctl.h" +#include "gasket_constants.h" +#include "gasket_core.h" +#include "gasket_interrupt.h" +#include "gasket_logging.h" +#include "gasket_page_table.h" +#include +#include + +#ifdef GASKET_KERNEL_TRACE_SUPPORT +#define CREATE_TRACE_POINTS +#include +#else +#define trace_gasket_ioctl_entry(x, ...) +#define trace_gasket_ioctl_exit(x) +#define trace_gasket_ioctl_integer_data(x) +#define trace_gasket_ioctl_eventfd_data(x, ...) +#define trace_gasket_ioctl_page_table_data(x, ...) +#define trace_gasket_ioctl_config_coherent_allocator(x, ...) +#endif + +static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd); +static int gasket_set_event_fd(struct gasket_dev *dev, ulong arg); +static int gasket_read_page_table_size( + struct gasket_dev *gasket_dev, ulong arg); +static int gasket_read_simple_page_table_size( + struct gasket_dev *gasket_dev, ulong arg); +static int gasket_partition_page_table( + struct gasket_dev *gasket_dev, ulong arg); +static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg); +static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg); +static int gasket_config_coherent_allocator( + struct gasket_dev *gasket_dev, ulong arg); + +/* + * standard ioctl dispatch function. + * @filp: File structure pointer describing this node usage session. + * @cmd: ioctl number to handle. + * @arg: ioctl-specific data pointer. + * + * Standard ioctl dispatcher; forwards operations to individual handlers. + */ +long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg) +{ + struct gasket_dev *gasket_dev; + int retval; + + gasket_dev = (struct gasket_dev *)filp->private_data; + trace_gasket_ioctl_entry(gasket_dev->dev_info.name, cmd); + + if (gasket_get_ioctl_permissions_cb(gasket_dev)) { + retval = gasket_get_ioctl_permissions_cb(gasket_dev)( + filp, cmd, arg); + if (retval < 0) { + trace_gasket_ioctl_exit(-EPERM); + return retval; + } else if (retval == 0) { + trace_gasket_ioctl_exit(-EPERM); + return -EPERM; + } + } else if (!gasket_ioctl_check_permissions(filp, cmd)) { + trace_gasket_ioctl_exit(-EPERM); + gasket_log_error(gasket_dev, "ioctl cmd=%x noperm.", cmd); + return -EPERM; + } + + /* Tracing happens in this switch statement for all ioctls with + * an integer argrument, but ioctls with a struct argument + * that needs copying and decoding, that tracing is done within + * the handler call. + */ + switch (cmd) { + case GASKET_IOCTL_RESET: + trace_gasket_ioctl_integer_data(arg); + retval = gasket_reset(gasket_dev, arg); + break; + case GASKET_IOCTL_SET_EVENTFD: + retval = gasket_set_event_fd(gasket_dev, arg); + break; + case GASKET_IOCTL_CLEAR_EVENTFD: + trace_gasket_ioctl_integer_data(arg); + retval = gasket_interrupt_clear_eventfd( + gasket_dev->interrupt_data, (int)arg); + break; + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + trace_gasket_ioctl_integer_data(arg); + retval = gasket_partition_page_table(gasket_dev, arg); + break; + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + trace_gasket_ioctl_integer_data(gasket_dev->num_page_tables); + if (copy_to_user((void __user *)arg, + &gasket_dev->num_page_tables, + sizeof(uint64_t))) + retval = -EFAULT; + else + retval = 0; + break; + case GASKET_IOCTL_PAGE_TABLE_SIZE: + retval = gasket_read_page_table_size(gasket_dev, arg); + break; + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + retval = gasket_read_simple_page_table_size(gasket_dev, arg); + break; + case GASKET_IOCTL_MAP_BUFFER: + retval = gasket_map_buffers(gasket_dev, arg); + break; + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: + retval = gasket_config_coherent_allocator(gasket_dev, arg); + break; + case GASKET_IOCTL_UNMAP_BUFFER: + retval = gasket_unmap_buffers(gasket_dev, arg); + break; + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + /* Clear interrupt counts doesn't take an arg, so use 0. */ + trace_gasket_ioctl_integer_data(0); + retval = gasket_interrupt_reset_counts(gasket_dev); + break; + default: + /* If we don't understand the ioctl, the best we can do is trace + * the arg. + */ + trace_gasket_ioctl_integer_data(arg); + gasket_log_warn( + gasket_dev, + "Unknown ioctl cmd=0x%x not caught by " + "gasket_is_supported_ioctl", + cmd); + retval = -EINVAL; + break; + } + + trace_gasket_ioctl_exit(retval); + return retval; +} + +/* + * Determines if an ioctl is part of the standard Gasket framework. + * @cmd: The ioctl number to handle. + * + * Returns 1 if the ioctl is supported and 0 otherwise. + */ +long gasket_is_supported_ioctl(uint cmd) +{ + switch (cmd) { + case GASKET_IOCTL_RESET: + case GASKET_IOCTL_SET_EVENTFD: + case GASKET_IOCTL_CLEAR_EVENTFD: + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_UNMAP_BUFFER: + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: + return 1; + default: + return 0; + } +} + +/* + * Permission checker for Gasket ioctls. + * @filp: File structure pointer describing this node usage session. + * @cmd: ioctl number to handle. + * + * Standard permissions checker. + */ +static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd) +{ + uint alive, root, device_owner; + fmode_t read, write; + struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; + + alive = (gasket_dev->status == GASKET_STATUS_ALIVE); + if (!alive) { + gasket_nodev_error( + "gasket_ioctl_check_permissions alive %d status %d.", + alive, gasket_dev->status); + } + + root = capable(CAP_SYS_ADMIN); + read = filp->f_mode & FMODE_READ; + write = filp->f_mode & FMODE_WRITE; + device_owner = (gasket_dev->dev_info.ownership.is_owned && + current->tgid == gasket_dev->dev_info.ownership.owner); + + switch (cmd) { + case GASKET_IOCTL_RESET: + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + return root || (write && device_owner); + + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + return root || read; + + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: + return alive && (root || (write && device_owner)); + + case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_UNMAP_BUFFER: + return alive && (root || (write && device_owner)); + + case GASKET_IOCTL_CLEAR_EVENTFD: + case GASKET_IOCTL_SET_EVENTFD: + return alive && (root || (write && device_owner)); + } + + return 0; /* unknown permissions */ +} + +/* + * Associate an eventfd with an interrupt. + * @gasket_dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to gasket_interrupt_eventfd struct in userspace. + */ +static int gasket_set_event_fd(struct gasket_dev *gasket_dev, ulong arg) +{ + struct gasket_interrupt_eventfd die; + + if (copy_from_user(&die, (void __user *)arg, + sizeof(struct gasket_interrupt_eventfd))) { + return -EFAULT; + } + + trace_gasket_ioctl_eventfd_data(die.interrupt, die.event_fd); + + return gasket_interrupt_set_eventfd( + gasket_dev->interrupt_data, die.interrupt, die.event_fd); +} + +/* + * Reads the size of the page table. + * @gasket_dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to gasket_page_table_ioctl struct in userspace. + */ +static int gasket_read_page_table_size(struct gasket_dev *gasket_dev, ulong arg) +{ + int ret = 0; + struct gasket_page_table_ioctl ibuf; + + if (copy_from_user(&ibuf, (void __user *)arg, + sizeof(struct gasket_page_table_ioctl))) + return -EFAULT; + + if (ibuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + ibuf.size = gasket_page_table_num_entries( + gasket_dev->page_table[ibuf.page_table_index]); + + trace_gasket_ioctl_page_table_data( + ibuf.page_table_index, ibuf.size, ibuf.host_address, + ibuf.device_address); + + if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf))) + return -EFAULT; + + return ret; +} + +/* + * Reads the size of the simple page table. + * @gasket_dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to gasket_page_table_ioctl struct in userspace. + */ +static int gasket_read_simple_page_table_size( + struct gasket_dev *gasket_dev, ulong arg) +{ + int ret = 0; + struct gasket_page_table_ioctl ibuf; + + if (copy_from_user(&ibuf, (void __user *)arg, + sizeof(struct gasket_page_table_ioctl))) + return -EFAULT; + + if (ibuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + ibuf.size = gasket_page_table_num_simple_entries( + gasket_dev->page_table[ibuf.page_table_index]); + + trace_gasket_ioctl_page_table_data( + ibuf.page_table_index, ibuf.size, ibuf.host_address, + ibuf.device_address); + + if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf))) + return -EFAULT; + + return ret; +} + +/* + * Sets the boundary between the simple and extended page tables. + * @gasket_dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to gasket_page_table_ioctl struct in userspace. + */ +static int gasket_partition_page_table(struct gasket_dev *gasket_dev, ulong arg) +{ + int ret; + struct gasket_page_table_ioctl ibuf; + uint max_page_table_size; + + if (copy_from_user(&ibuf, (void __user *)arg, + sizeof(struct gasket_page_table_ioctl))) + return -EFAULT; + + trace_gasket_ioctl_page_table_data( + ibuf.page_table_index, ibuf.size, ibuf.host_address, + ibuf.device_address); + + if (ibuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + max_page_table_size = gasket_page_table_max_size( + gasket_dev->page_table[ibuf.page_table_index]); + + if (ibuf.size > max_page_table_size) { + gasket_log_error( + gasket_dev, + "Partition request 0x%llx too large, max is 0x%x.", + ibuf.size, max_page_table_size); + return -EINVAL; + } + + mutex_lock(&gasket_dev->mutex); + + ret = gasket_page_table_partition( + gasket_dev->page_table[ibuf.page_table_index], ibuf.size); + mutex_unlock(&gasket_dev->mutex); + + return ret; +} + +/* + * Maps a userspace buffer to a device virtual address. + * @gasket_dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to a gasket_page_table_ioctl struct in userspace. + */ +static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg) +{ + struct gasket_page_table_ioctl ibuf; + + if (copy_from_user(&ibuf, (void __user *)arg, + sizeof(struct gasket_page_table_ioctl))) + return -EFAULT; + + trace_gasket_ioctl_page_table_data( + ibuf.page_table_index, ibuf.size, ibuf.host_address, + ibuf.device_address); + + if (ibuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + if (gasket_page_table_are_addrs_bad( + gasket_dev->page_table[ibuf.page_table_index], + ibuf.host_address, ibuf.device_address, ibuf.size)) + return -EINVAL; + + return gasket_page_table_map( + gasket_dev->page_table[ibuf.page_table_index], + ibuf.host_address, ibuf.device_address, ibuf.size / PAGE_SIZE); +} + +/* + * Unmaps a userspace buffer from a device virtual address. + * @gasket_dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to a gasket_page_table_ioctl struct in userspace. + */ +static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg) +{ + struct gasket_page_table_ioctl ibuf; + + if (copy_from_user(&ibuf, (void __user *)arg, + sizeof(struct gasket_page_table_ioctl))) + return -EFAULT; + + trace_gasket_ioctl_page_table_data( + ibuf.page_table_index, ibuf.size, ibuf.host_address, + ibuf.device_address); + + if (ibuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + if (gasket_page_table_is_dev_addr_bad( + gasket_dev->page_table[ibuf.page_table_index], + ibuf.device_address, ibuf.size)) + return -EINVAL; + + gasket_page_table_unmap(gasket_dev->page_table[ibuf.page_table_index], + ibuf.device_address, ibuf.size / PAGE_SIZE); + + return 0; +} + +/* + * Tell the driver to reserve structures for coherent allocation, and allocate + * or free the corresponding memory. + * @dev: Pointer to the current gasket_dev we're using. + * @arg: Pointer to a gasket_coherent_alloc_config_ioctl struct in userspace. + */ +static int gasket_config_coherent_allocator( + struct gasket_dev *gasket_dev, ulong arg) +{ + int ret; + struct gasket_coherent_alloc_config_ioctl ibuf; + + if (copy_from_user(&ibuf, (void __user *)arg, + sizeof(struct gasket_coherent_alloc_config_ioctl))) + return -EFAULT; + + trace_gasket_ioctl_config_coherent_allocator( + ibuf.enable, ibuf.size, ibuf.dma_address); + + if (ibuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + if (ibuf.size > PAGE_SIZE * MAX_NUM_COHERENT_PAGES) { + ibuf.size = PAGE_SIZE * MAX_NUM_COHERENT_PAGES; + return -ENOMEM; + } + + if (ibuf.enable == 0) { + ret = gasket_free_coherent_memory( + gasket_dev, ibuf.size, ibuf.dma_address, + ibuf.page_table_index); + } else { + ret = gasket_alloc_coherent_memory( + gasket_dev, ibuf.size, &ibuf.dma_address, + ibuf.page_table_index); + } + if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf))) + return -EFAULT; + + return ret; +} diff --git a/gasket_ioctl.h b/gasket_ioctl.h new file mode 100644 index 0000000..df86800 --- /dev/null +++ b/gasket_ioctl.h @@ -0,0 +1,35 @@ +/* Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __GASKET_IOCTL_H__ +#define __GASKET_IOCTL_H__ + +#include "gasket_core.h" + +/* + * Handle Gasket common ioctls. + * @filp: Pointer to the ioctl's file. + * @cmd: Ioctl command. + * @arg: Ioctl argument pointer. + * + * Returns 0 on success and nonzero on failure. + */ +long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg); + +/* + * Determines if an ioctl is part of the standard Gasket framework. + * @cmd: The ioctl number to handle. + * + * Returns 1 if the ioctl is supported and 0 otherwise. + */ +long gasket_is_supported_ioctl(uint cmd); + +#endif diff --git a/gasket_logging.h b/gasket_logging.h new file mode 100644 index 0000000..fa17b4a --- /dev/null +++ b/gasket_logging.h @@ -0,0 +1,71 @@ +/* Common logging utilities for the Gasket driver framework. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include + +#ifndef _GASKET_LOGGING_H_ +#define _GASKET_LOGGING_H_ + +/* Base macro; other logging can/should be built on top of this. */ +#define gasket_dev_log(level, device, pci_dev, format, arg...) \ + if (false) { \ + if (pci_dev) { \ + dev_##level(&(pci_dev)->dev, "%s: " format "\n", \ + __func__, ##arg); \ + } else { \ + gasket_nodev_log(level, format, ##arg); \ + } \ + } + +/* "No-device" logging macros. */ +#define gasket_nodev_log(level, format, arg...) \ + if (false) pr_##level("gasket: %s: " format "\n", __func__, ##arg) + +#define gasket_nodev_debug(format, arg...) \ + if (false) gasket_nodev_log(debug, format, ##arg) + +#define gasket_nodev_info(format, arg...) \ + if (false) gasket_nodev_log(info, format, ##arg) + +#define gasket_nodev_warn(format, arg...) \ + if (false) gasket_nodev_log(warn, format, ##arg) + +#define gasket_nodev_error(format, arg...) \ + if (false) gasket_nodev_log(err, format, ##arg) + +/* gasket_dev logging macros */ +#define gasket_log_info(gasket_dev, format, arg...) \ + if (false) gasket_dev_log(info, (gasket_dev)->dev_info.device, \ + (gasket_dev)->pci_dev, format, ##arg) + +#define gasket_log_warn(gasket_dev, format, arg...) \ + if (false) gasket_dev_log(warn, (gasket_dev)->dev_info.device, \ + (gasket_dev)->pci_dev, format, ##arg) + +#define gasket_log_error(gasket_dev, format, arg...) \ + if (false) gasket_dev_log(err, (gasket_dev)->dev_info.device, \ + (gasket_dev)->pci_dev, format, ##arg) + +#define gasket_log_debug(gasket_dev, format, arg...) \ + if (false){ \ + if ((gasket_dev)->pci_dev) { \ + dev_dbg(&((gasket_dev)->pci_dev)->dev, "%s: " format \ + "\n", __func__, ##arg); \ + } else { \ + gasket_nodev_log(debug, format, ##arg); \ + } \ + } + +#endif diff --git a/gasket_page_table.c b/gasket_page_table.c new file mode 100644 index 0000000..6dc1050 --- /dev/null +++ b/gasket_page_table.c @@ -0,0 +1,1771 @@ +/* Implementation of Gasket page table support. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* + * Implementation of Gasket page table support. + * + * This file assumes 4kB pages throughout; can be factored out when necessary. + * + * Address format is as follows: + * Simple addresses - those whose containing pages are directly placed in the + * device's address translation registers - are laid out as: + * [ 63 - 40: Unused | 39 - 28: 0 | 27 - 12: page index | 11 - 0: page offset ] + * page index: The index of the containing page in the device's address + * translation registers. + * page offset: The index of the address into the containing page. + * + * Extended address - those whose containing pages are contained in a second- + * level page table whose address is present in the device's address translation + * registers - are laid out as: + * [ 63 - 40: Unused | 39: flag | 38 - 37: 0 | 36 - 21: dev/level 0 index | + * 20 - 12: host/level 1 index | 11 - 0: page offset ] + * flag: Marker indicating that this is an extended address. Always 1. + * dev index: The index of the first-level page in the device's extended + * address translation registers. + * host index: The index of the containing page in the [host-resident] second- + * level page table. + * page offset: The index of the address into the containing [second-level] + * page. + */ +#include "gasket_page_table.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "gasket_constants.h" +#include "gasket_core.h" +#include "gasket_logging.h" + +/* Constants & utility macros */ +/* The number of pages that can be mapped into each second-level page table. */ +#define GASKET_PAGES_PER_SUBTABLE 512 + +/* The starting position of the page index in a simple virtual address. */ +#define GASKET_SIMPLE_PAGE_SHIFT 12 + +/* Flag indicating that a [device] slot is valid for use. */ +#define GASKET_VALID_SLOT_FLAG 1 + +/* + * The starting position of the level 0 page index (i.e., the entry in the + * device's extended address registers) in an extended address. + * Also can be thought of as (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)), + * or (12 + 9). + */ +#define GASKET_EXTENDED_LVL0_SHIFT 21 + +/* + * Number of first level pages that Gasket chips support. Equivalent to + * log2(NUM_LVL0_PAGE_TABLES) + * + * At a maximum, allowing for a 34 bits address space (or 16GB) + * = GASKET_EXTENDED_LVL0_WIDTH + (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE) + * or, = 13 + 9 + 12 + */ +#define GASKET_EXTENDED_LVL0_WIDTH 13 + +/* + * The starting position of the level 1 page index (i.e., the entry in the + * host second-level/sub- table) in an extended address. + */ +#define GASKET_EXTENDED_LVL1_SHIFT 12 + +/* Page-table specific error logging. */ +#define gasket_pg_tbl_error(pg_tbl, format, arg...) \ + gasket_dev_log(err, (pg_tbl)->device, (struct pci_dev *)NULL, format, \ + ##arg) + +/* Type declarations */ +/* Valid states for a struct gasket_page_table_entry. */ +enum pte_status { + PTE_FREE, + PTE_INUSE, +}; + +/* + * Mapping metadata for a single page. + * + * In this file, host-side page table entries are referred to as that (or PTEs). + * Where device vs. host entries are differentiated, device-side or -visible + * entries are called "slots". A slot may be either an entry in the device's + * address translation table registers or an entry in a second-level page + * table ("subtable"). + * + * The full data in this structure is visible on the host [of course]. Only + * the address contained in dma_addr is communicated to the device; that points + * to the actual page mapped and described by this structure. + */ +struct gasket_page_table_entry { + /* The status of this entry/slot: free or in use. */ + enum pte_status status; + + /* Address of the page in DMA space. */ + dma_addr_t dma_addr; + + /* Linux page descriptor for the page described by this structure. */ + struct page *page; + + /* + * Index for alignment into host vaddrs. + * When a user specifies a host address for a mapping, that address may + * not be page-aligned. Offset is the index into the containing page of + * the host address (i.e., host_vaddr & (PAGE_SIZE - 1)). + * This is necessary for translating between user-specified addresses + * and page-aligned addresses. + */ + int offset; + + /* + * If this is an extended and first-level entry, sublevel points + * to the second-level entries underneath this entry. + */ + struct gasket_page_table_entry *sublevel; +}; + +/* + * Maintains virtual to physical address mapping for a coherent page that is + * allocated by this module for a given device. + * Note that coherent pages mappings virt mapping cannot be tracked by the + * Linux kernel, and coherent pages don't have a struct page associated, + * hence Linux kernel cannot perform a get_user_page_xx() on a phys address + * that was allocated coherent. + * This structure trivially implements this mechanism. + */ +struct gasket_coherent_page_entry { + /* Phys address, dma'able by the owner device */ + dma_addr_t paddr; + + /* Kernel virtual address */ + u64 user_virt; + + /* User virtual address that was mapped by the mmap kernel subsystem */ + u64 kernel_virt; + + /* + * Whether this page has been mapped into a user land process virtual + * space + */ + u32 in_use; +}; + +/* + * [Host-side] page table descriptor. + * + * This structure tracks the metadata necessary to manage both simple and + * extended page tables. + */ +struct gasket_page_table { + /* The config used to create this page table. */ + struct gasket_page_table_config config; + + /* The number of simple (single-level) entries in the page table. */ + uint num_simple_entries; + + /* The number of extended (two-level) entries in the page table. */ + uint num_extended_entries; + + /* Array of [host-side] page table entries. */ + struct gasket_page_table_entry *entries; + + /* Number of actively mapped kernel pages in this table. */ + uint num_active_pages; + + /* Device register: base of/first slot in the page table. */ + u64 __iomem *base_slot; + + /* Device register: holds the offset indicating the start of the + * extended address region of the device's address translation table. + */ + u64 __iomem *extended_offset_reg; + + /* Device structure for the underlying device. Only used for logging. */ + struct device *device; + + /* PCI system descriptor for the underlying device. */ + struct pci_dev *pci_dev; + + /* Location of the extended address bit for this Gasket device. */ + u64 extended_flag; + + /* Mutex to protect page table internals. */ + struct mutex mutex; + + /* Number of coherent pages accessible thru by this page table */ + int num_coherent_pages; + + /* + * List of coherent memory (physical) allocated for a device. + * + * This structure also remembers the user virtual mapping, this is + * hacky, but we need to do this because the kernel doesn't keep track + * of the user coherent pages (pfn pages), and virt to coherent page + * mapping. + * TODO: use find_vma() APIs to convert host address to vm_area, to + * dma_addr_t instead of storing user virtu address in + * gasket_coherent_page_entry + * + * Note that the user virtual mapping is created by the driver, in + * gasket_mmap function, so user_virt belongs in the driver anyhow. + */ + struct gasket_coherent_page_entry *coherent_pages; + + /* + * Whether the page table uses arch specific dma_ops or + * whether the driver is supplying its own. + */ + bool dma_ops; +}; + +/* Mapping declarations */ +static int gasket_map_simple_pages( + struct gasket_page_table *pg_tbl, ulong host_addr, + ulong dev_addr, uint num_pages); +static int gasket_map_extended_pages( + struct gasket_page_table *pg_tbl, ulong host_addr, + ulong dev_addr, uint num_pages); +static int gasket_perform_mapping( + struct gasket_page_table *pg_tbl, + struct gasket_page_table_entry *pte_base, u64 __iomem *att_base, + ulong host_addr, uint num_pages, int is_simple_mapping); + +static int gasket_alloc_simple_entries( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); +static int gasket_alloc_extended_entries( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries); +static int gasket_alloc_extended_subtable( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, + u64 __iomem *att_reg); + +/* Unmapping declarations */ +static void gasket_page_table_unmap_nolock( + struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages); +static void gasket_page_table_unmap_all_nolock( + struct gasket_page_table *pg_tbl); +static void gasket_unmap_simple_pages( + struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages); +static void gasket_unmap_extended_pages( + struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages); +static void gasket_perform_unmapping( + struct gasket_page_table *pg_tbl, + struct gasket_page_table_entry *pte_base, u64 __iomem *att_base, + uint num_pages, int is_simple_mapping); + +static void gasket_free_extended_subtable( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, + u64 __iomem *att_reg); +static int gasket_release_page(struct page *page); + +/* Other/utility declarations */ +static inline int gasket_addr_is_simple( + struct gasket_page_table *pg_tbl, ulong addr); +static int gasket_is_simple_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); +static int gasket_is_extended_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); +static int gasket_is_pte_range_free( + struct gasket_page_table_entry *pte, uint num_entries); +static void gasket_page_table_garbage_collect_nolock( + struct gasket_page_table *pg_tbl); + +/* Address format declarations */ +static ulong gasket_components_to_dev_address( + struct gasket_page_table *pg_tbl, int is_simple, uint page_index, + uint offset); +static int gasket_simple_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr); +static ulong gasket_extended_lvl0_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr); +static ulong gasket_extended_lvl1_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr); + +static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr); + +/* Public/exported functions */ +/* See gasket_page_table.h for description. */ +int gasket_page_table_init( + struct gasket_page_table **ppg_tbl, + const struct gasket_bar_data *bar_data, + const struct gasket_page_table_config *page_table_config, + struct device *device, struct pci_dev *pci_dev, bool has_dma_ops) +{ + ulong bytes; + struct gasket_page_table *pg_tbl; + ulong total_entries = page_table_config->total_entries; + + /* + * TODO: Verify config->total_entries against value read from the + * hardware register that contains the page table size. + */ + if (total_entries == ULONG_MAX) { + gasket_nodev_error( + "Error reading page table size. " + "Initializing page table with size 0."); + total_entries = 0; + } + + gasket_nodev_debug( + "Attempting to initialize page table of size 0x%lx.", + total_entries); + + gasket_nodev_debug( + "Table has base reg 0x%x, extended offset reg 0x%x.", + page_table_config->base_reg, + page_table_config->extended_reg); + + *ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL); + if (!*ppg_tbl) { + gasket_nodev_error("No memory for page table."); + return -ENOMEM; + } + + pg_tbl = *ppg_tbl; + bytes = total_entries * sizeof(struct gasket_page_table_entry); + if (bytes != 0) { + pg_tbl->entries = vmalloc(bytes); + if (!pg_tbl->entries) { + gasket_nodev_error( + "No memory for address translation metadata."); + kfree(pg_tbl); + *ppg_tbl = NULL; + return -ENOMEM; + } + memset(pg_tbl->entries, 0, bytes); + } + + mutex_init(&pg_tbl->mutex); + memcpy(&pg_tbl->config, page_table_config, sizeof(*page_table_config)); + if (pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_NORMAL || + pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_SIMPLE) { + pg_tbl->num_simple_entries = total_entries; + pg_tbl->num_extended_entries = 0; + pg_tbl->extended_flag = 1ull << page_table_config->extended_bit; + } else { + pg_tbl->num_simple_entries = 0; + pg_tbl->num_extended_entries = total_entries; + pg_tbl->extended_flag = 0; + } + pg_tbl->num_active_pages = 0; + pg_tbl->base_slot = (u64 __iomem *)&( + bar_data->virt_base[page_table_config->base_reg]); + pg_tbl->extended_offset_reg = (u64 __iomem *)&( + bar_data->virt_base[page_table_config->extended_reg]); + pg_tbl->device = device; + pg_tbl->pci_dev = pci_dev; + pg_tbl->dma_ops = has_dma_ops; + + gasket_nodev_debug("Page table initialized successfully."); + + return 0; +} + +/* See gasket_page_table.h for description. */ +void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl) +{ + /* Deallocate free second-level tables. */ + gasket_page_table_garbage_collect(pg_tbl); + + /* TODO: Check that all PTEs have been freed? */ + + vfree(pg_tbl->entries); + pg_tbl->entries = NULL; + + kfree(pg_tbl); +} + +/* See gasket_page_table.h for description. */ +int gasket_page_table_partition( + struct gasket_page_table *pg_tbl, uint num_simple_entries) +{ + int i, start; + + mutex_lock(&pg_tbl->mutex); + if (num_simple_entries > pg_tbl->config.total_entries) { + mutex_unlock(&pg_tbl->mutex); + return -EINVAL; + } + + gasket_page_table_garbage_collect_nolock(pg_tbl); + + start = min(pg_tbl->num_simple_entries, num_simple_entries); + + for (i = start; i < pg_tbl->config.total_entries; i++) { + if (pg_tbl->entries[i].status != PTE_FREE) { + gasket_pg_tbl_error(pg_tbl, "entry %d is not free", i); + mutex_unlock(&pg_tbl->mutex); + return -EBUSY; + } + } + + pg_tbl->num_simple_entries = num_simple_entries; + pg_tbl->num_extended_entries = + pg_tbl->config.total_entries - num_simple_entries; + writeq(num_simple_entries, pg_tbl->extended_offset_reg); + + mutex_unlock(&pg_tbl->mutex); + return 0; +} +EXPORT_SYMBOL(gasket_page_table_partition); + +/* + * See gasket_page_table.h for general description. + * + * gasket_page_table_map calls either gasket_map_simple_pages() or + * gasket_map_extended_pages() to actually perform the mapping. + * + * The page table mutex is held for the entire operation. + */ +int gasket_page_table_map( + struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + uint num_pages) +{ + int ret; + + if (!num_pages) + return 0; + + mutex_lock(&pg_tbl->mutex); + + if (gasket_addr_is_simple(pg_tbl, dev_addr)) { + ret = gasket_map_simple_pages( + pg_tbl, host_addr, dev_addr, num_pages); + } else { + ret = gasket_map_extended_pages( + pg_tbl, host_addr, dev_addr, num_pages); + } + + mutex_unlock(&pg_tbl->mutex); + + gasket_nodev_debug( + "gasket_page_table_map done: ha %llx daddr %llx num %d, " + "ret %d\n", + (unsigned long long)host_addr, + (unsigned long long)dev_addr, num_pages, ret); + return ret; +} +EXPORT_SYMBOL(gasket_page_table_map); + +/* + * See gasket_page_table.h for general description. + * + * gasket_page_table_unmap takes the page table lock and calls either + * gasket_unmap_simple_pages() or gasket_unmap_extended_pages() to + * actually unmap the pages from device space. + * + * The page table mutex is held for the entire operation. + */ +void gasket_page_table_unmap( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + if (!num_pages) + return; + + mutex_lock(&pg_tbl->mutex); + gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); + mutex_unlock(&pg_tbl->mutex); +} +EXPORT_SYMBOL(gasket_page_table_unmap); + +static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl) +{ + gasket_unmap_simple_pages( + pg_tbl, gasket_components_to_dev_address(pg_tbl, 1, 0, 0), + pg_tbl->num_simple_entries); + gasket_unmap_extended_pages( + pg_tbl, gasket_components_to_dev_address(pg_tbl, 0, 0, 0), + pg_tbl->num_extended_entries * GASKET_PAGES_PER_SUBTABLE); +} + +/* See gasket_page_table.h for description. */ +void gasket_page_table_unmap_all(struct gasket_page_table *pg_tbl) +{ + mutex_lock(&pg_tbl->mutex); + gasket_page_table_unmap_all_nolock(pg_tbl); + mutex_unlock(&pg_tbl->mutex); +} +EXPORT_SYMBOL(gasket_page_table_unmap_all); + +/* See gasket_page_table.h for description. */ +void gasket_page_table_reset(struct gasket_page_table *pg_tbl) +{ + mutex_lock(&pg_tbl->mutex); + gasket_page_table_unmap_all_nolock(pg_tbl); + writeq(pg_tbl->config.total_entries, pg_tbl->extended_offset_reg); + mutex_unlock(&pg_tbl->mutex); +} + +/* See gasket_page_table.h for description. */ +void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl) +{ + mutex_lock(&pg_tbl->mutex); + gasket_page_table_garbage_collect_nolock(pg_tbl); + mutex_unlock(&pg_tbl->mutex); +} + +/* See gasket_page_table.h for description. */ +int gasket_page_table_lookup_page( + struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage, + ulong *poffset) +{ + uint page_num; + struct gasket_page_table_entry *pte; + + mutex_lock(&pg_tbl->mutex); + if (gasket_addr_is_simple(pg_tbl, dev_addr)) { + page_num = gasket_simple_page_idx(pg_tbl, dev_addr); + if (page_num >= pg_tbl->num_simple_entries) + goto fail; + + pte = pg_tbl->entries + page_num; + if (pte->status != PTE_INUSE) + goto fail; + } else { + /* Find the level 0 entry, */ + page_num = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + if (page_num >= pg_tbl->num_extended_entries) + goto fail; + + pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num; + if (pte->status != PTE_INUSE) + goto fail; + + /* and its contained level 1 entry. */ + page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pte->sublevel + page_num; + if (pte->status != PTE_INUSE) + goto fail; + } + + *ppage = pte->page; + *poffset = pte->offset; + mutex_unlock(&pg_tbl->mutex); + return 0; + +fail: + *ppage = NULL; + *poffset = 0; + mutex_unlock(&pg_tbl->mutex); + return -1; +} + +/* See gasket_page_table.h for description. */ +int gasket_page_table_are_addrs_bad( + struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + ulong bytes) +{ + if (host_addr & (PAGE_SIZE - 1)) { + gasket_pg_tbl_error( + pg_tbl, + "host mapping address 0x%lx must be page aligned", + host_addr); + return 1; + } + + return gasket_page_table_is_dev_addr_bad(pg_tbl, dev_addr, bytes); +} +EXPORT_SYMBOL(gasket_page_table_are_addrs_bad); + +/* See gasket_page_table.h for description. */ +int gasket_page_table_is_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes) +{ + uint num_pages = bytes / PAGE_SIZE; + + if (bytes & (PAGE_SIZE - 1)) { + gasket_pg_tbl_error( + pg_tbl, + "mapping size 0x%lX must be page aligned", bytes); + return 1; + } + + if (num_pages == 0) { + gasket_pg_tbl_error( + pg_tbl, + "requested mapping is less than one page: %lu / %lu", + bytes, PAGE_SIZE); + return 1; + } + + if (gasket_addr_is_simple(pg_tbl, dev_addr)) + return gasket_is_simple_dev_addr_bad( + pg_tbl, dev_addr, num_pages); + else + return gasket_is_extended_dev_addr_bad( + pg_tbl, dev_addr, num_pages); +} +EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad); + +/* See gasket_page_table.h for description. */ +uint gasket_page_table_max_size(struct gasket_page_table *page_table) +{ + if (!page_table) { + gasket_nodev_error("Passed a null page table."); + return 0; + } + return page_table->config.total_entries; +} +EXPORT_SYMBOL(gasket_page_table_max_size); + +/* See gasket_page_table.h for description. */ +uint gasket_page_table_num_entries(struct gasket_page_table *pg_tbl) +{ + if (!pg_tbl) { + gasket_nodev_error("Passed a null page table."); + return 0; + } + + return pg_tbl->num_simple_entries + pg_tbl->num_extended_entries; +} +EXPORT_SYMBOL(gasket_page_table_num_entries); + +/* See gasket_page_table.h for description. */ +uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl) +{ + if (!pg_tbl) { + gasket_nodev_error("Passed a null page table."); + return 0; + } + + return pg_tbl->num_simple_entries; +} +EXPORT_SYMBOL(gasket_page_table_num_simple_entries); + +/* See gasket_page_table.h for description. */ +uint gasket_page_table_num_extended_entries(struct gasket_page_table *pg_tbl) +{ + if (!pg_tbl) { + gasket_nodev_error("Passed a null page table."); + return 0; + } + + return pg_tbl->num_extended_entries; +} +EXPORT_SYMBOL(gasket_page_table_num_extended_entries); + +uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl) +{ + if (!pg_tbl) { + gasket_nodev_error("Passed a null page table."); + return 0; + } + + return pg_tbl->num_active_pages; +} +EXPORT_SYMBOL(gasket_page_table_num_active_pages); + +/* See gasket_page_table.h */ +int gasket_page_table_system_status(struct gasket_page_table *page_table) +{ + if (!page_table) { + gasket_nodev_error("Passed a null page table."); + return GASKET_STATUS_LAMED; + } + + if (gasket_page_table_num_entries(page_table) == 0) { + gasket_nodev_error("Page table size is 0."); + return GASKET_STATUS_LAMED; + } + + return GASKET_STATUS_ALIVE; +} + +/* Internal functions */ + +/* Mapping functions */ +/* + * Allocate and map pages to simple addresses. + * @pg_tbl: Gasket page table pointer. + * @host_addr: Starting host virtual memory address of the pages. + * @dev_addr: Starting device address of the pages. + * @cnt: Count of the number of device pages to map. + * + * Description: gasket_map_simple_pages calls gasket_simple_alloc_pages() to + * allocate the page table slots, then calls + * gasket_perform_mapping() to actually do the work of mapping the + * pages into the the simple page table (device translation table + * registers). + * + * The sd_mutex must be held when gasket_map_simple_pages() is + * called. + * + * Returns 0 if successful or a non-zero error number otherwise. + * If there is an error, no pages are mapped. + */ +static int gasket_map_simple_pages( + struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + uint num_pages) +{ + int ret; + uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); + + ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); + if (ret) { + gasket_pg_tbl_error( + pg_tbl, + "page table slots %u (@ 0x%lx) to %u are not available", + slot_idx, dev_addr, slot_idx + num_pages - 1); + return ret; + } + + ret = gasket_perform_mapping( + pg_tbl, pg_tbl->entries + slot_idx, + pg_tbl->base_slot + slot_idx, host_addr, num_pages, 1); + + if (ret) { + gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); + gasket_pg_tbl_error(pg_tbl, "gasket_perform_mapping %d.", ret); + } + return ret; +} + +/* + * gasket_map_extended_pages - Get and map buffers to extended addresses. + * @pg_tbl: Gasket page table pointer. + * @host_addr: Starting host virtual memory address of the pages. + * @dev_addr: Starting device address of the pages. + * @num_pages: The number of device pages to map. + * + * Description: gasket_map_extended_buffers calls + * gasket_alloc_extended_entries() to allocate the page table + * slots, then loops over the level 0 page table entries, and for + * each calls gasket_perform_mapping() to map the buffers into the + * level 1 page table for that level 0 entry. + * + * The page table mutex must be held when + * gasket_map_extended_pages() is called. + * + * Returns 0 if successful or a non-zero error number otherwise. + * If there is an error, no pages are mapped. + */ +static int gasket_map_extended_pages( + struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + uint num_pages) +{ + int ret; + ulong dev_addr_end; + uint slot_idx, remain, len; + struct gasket_page_table_entry *pte; + u64 __iomem *slot_base; + + ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages); + if (ret) { + dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; + gasket_pg_tbl_error( + pg_tbl, + "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are " + "not available", + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), + dev_addr, + gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end), + gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end)); + return ret; + } + + remain = num_pages; + slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pg_tbl->entries + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + while (remain > 0) { + len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); + + slot_base = + (u64 __iomem *)(page_address(pte->page) + pte->offset); + ret = gasket_perform_mapping( + pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, + host_addr, len, 0); + if (ret) { + gasket_page_table_unmap_nolock( + pg_tbl, dev_addr, num_pages); + return ret; + } + + remain -= len; + slot_idx = 0; + pte++; + host_addr += len * PAGE_SIZE; + } + + return 0; +} + +/* + * TODO: dma_map_page() is not plugged properly when running under qemu. i.e. + * dma_ops are not set properly, which causes the kernel to assert. + * + * This temporary hack allows the driver to work on qemu, but need to be fixed: + * - either manually set the dma_ops for the architecture (which incidentally + * can't be done in an out-of-tree module) - or get qemu to fill the device tree + * properly so as linux plug the proper dma_ops or so as the driver can detect + * that it is runnig on qemu + */ +static inline dma_addr_t _no_op_dma_map_page( + struct device *dev, struct page *page, size_t offset, size_t size, + enum dma_data_direction dir) +{ + /* + * struct dma_map_ops *ops = get_dma_ops(dev); + * dma_addr_t addr; + * + * kmemcheck_mark_initialized(page_address(page) + offset, size); + * BUG_ON(!valid_dma_direction(dir)); + * addr = ops->map_page(dev, page, offset, size, dir, NULL); + * debug_dma_map_page(dev, page, offset, size, dir, addr, false); + */ + + return page_to_phys(page); +} + +/* + * Get and map last level page table buffers. + * @pg_tbl: Gasket page table pointer. + * @ptes: Array of page table entries to describe this mapping, one per + * page to map. + * @slots: Location(s) to write device-mapped page address. If this is a simple + * mapping, these will be address translation registers. If this is + * an extended mapping, these will be within a second-level page table + * allocated by the host and so must have their __iomem attribute + * casted away. + * @host_addr: Starting [host] virtual memory address of the buffers. + * @num_pages: The number of device pages to map. + * @is_simple_mapping: 1 if this is a simple mapping, 0 otherwise. + * + * Description: gasket_perform_mapping calls get_user_pages() to get pages + * of user memory and pin them. It then calls dma_map_page() to + * map them for DMA. Finally, the mapped DMA addresses are written + * into the page table. + * + * This function expects that the page table entries are + * already allocated. The level argument determines how the + * final page table entries are written: either into PCIe memory + * mapped space for a level 0 page table or into kernel memory + * for a level 1 page table. + * + * The page pointers are saved for later releasing the pages. + * + * Returns 0 if successful or a non-zero error number otherwise. + */ +static int gasket_perform_mapping( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, + u64 __iomem *slots, ulong host_addr, uint num_pages, + int is_simple_mapping) +{ + int ret; + ulong offset; + struct page *page; + dma_addr_t dma_addr; + ulong page_addr; + int i; + + for (i = 0; i < num_pages; i++) { + page_addr = host_addr + i * PAGE_SIZE; + offset = page_addr & (PAGE_SIZE - 1); + gasket_nodev_debug("gasket_perform_mapping i %d\n", i); + if (is_coherent(pg_tbl, host_addr)) { + u64 off = + (u64)host_addr - + (u64)pg_tbl->coherent_pages[0].user_virt; + ptes[i].page = 0; + ptes[i].offset = offset; + ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + + off + i * PAGE_SIZE; + } else { + ret = get_user_pages_fast( + page_addr - offset, 1, 1, &page); + + if (ret <= 0) { + gasket_pg_tbl_error( + pg_tbl, + "get user pages failed for addr=0x%lx, " + "offset=0x%lx [ret=%d]", + page_addr, offset, ret); + return ret ? ret : -ENOMEM; + } + ++pg_tbl->num_active_pages; + + ptes[i].page = page; + ptes[i].offset = offset; + + /* Map the page into DMA space. */ + if (pg_tbl->dma_ops) { + /* hook in to kernel map functions */ + ptes[i].dma_addr = dma_map_page(pg_tbl->device, + page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + } else { + ptes[i].dma_addr = _no_op_dma_map_page( + pg_tbl->device, page, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + } + + gasket_nodev_debug( + " gasket_perform_mapping dev %p " + "i %d pte %p pfn %p -> mapped %llx\n", + pg_tbl->device, i, &ptes[i], + (void *)page_to_pfn(page), + (unsigned long long)ptes[i].dma_addr); + + if (ptes[i].dma_addr == -1) { + gasket_nodev_error( + "gasket_perform_mapping i %d" + " -> fail to map page %llx " + "[pfn %p ohys %p]\n", + i, + (unsigned long long)ptes[i].dma_addr, + (void *)page_to_pfn(page), + (void *)page_to_phys(page)); + return -1; + } + /* Wait until the page is mapped. */ + mb(); + } + + /* Make the DMA-space address available to the device. */ + dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG; + + if (is_simple_mapping) { + writeq(dma_addr, &slots[i]); + } else { + ((u64 __force *)slots)[i] = dma_addr; + /* Extended page table vectors are in DRAM, + * and so need to be synced each time they are updated. + */ + dma_map_single(pg_tbl->device, + (void *)&((u64 __force *)slots)[i], + sizeof(u64), DMA_TO_DEVICE); + } + ptes[i].status = PTE_INUSE; + } + return 0; +} + +/** + * Allocate page table entries in a simple table. + * @pg_tbl: Gasket page table pointer. + * @dev_addr: Starting device address for the (eventual) mappings. + * @num_pages: Count of pages to be mapped. + * + * Description: gasket_alloc_simple_entries checks to see if a range of page + * table slots are available. As long as the sd_mutex is + * held, the slots will be available. + * + * The page table mutex must be held when + * gasket_alloc_simple entries() is called. + * + * Returns 0 if successful, or non-zero if the requested device + * addresses are not available. + */ +static int gasket_alloc_simple_entries( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + if (!gasket_is_pte_range_free( + pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr), + num_pages)) + return -EBUSY; + + return 0; +} + +/** + * Allocate slots in an extended page table. + * @pg_tbl: Gasket page table pointer. + * @dev_addr: Starting device address for the (eventual) mappings. + * @num_pages: Count of pages to be mapped. + * + * Description: gasket_alloc_extended_entries checks to see if a range of page + * table slots are available. If necessary, memory is allocated for + * second level page tables. + * + * Note that memory for second level page tables is allocated + * as needed, but that memory is only freed on the final close + * of the device file, when the page tables are repartitioned, + * or the the device is removed. If there is an error or if + * the full range of slots is not available, any memory + * allocated for second level page tables remains allocated + * until final close, repartition, or device removal. + * + * The page table mutex must be held when + * gasket_alloc_extended_entries() is called. + * + * Returns 0 if successful, or non-zero if the slots are + * not available. + */ +static int gasket_alloc_extended_entries( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries) +{ + int ret = 0; + uint remain, subtable_slot_idx, len; + struct gasket_page_table_entry *pte; + u64 __iomem *slot; + + remain = num_entries; + subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pg_tbl->entries + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + slot = pg_tbl->base_slot + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + while (remain > 0) { + len = min(remain, + GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx); + + if (pte->status == PTE_FREE) { + ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); + if (ret) { + gasket_pg_tbl_error( + pg_tbl, + "no memory for extended addr subtable"); + return ret; + } + } else { + if (!gasket_is_pte_range_free( + pte->sublevel + subtable_slot_idx, len)) + return -EBUSY; + } + + remain -= len; + subtable_slot_idx = 0; + pte++; + slot++; + } + + return 0; +} + +/** + * Allocate a second level page table. + * @pg_tbl: Gasket page table pointer. + * @pte: Extended page table entry under/for which to allocate a second level. + * @slot: [Device] slot corresponding to pte. + * + * Description: Allocate the memory for a second level page table (subtable) at + * the given level 0 entry. Then call dma_map_page() to map the + * second level page table for DMA. Finally, write the + * mapped DMA address into the device page table. + * + * The page table mutex must be held when + * gasket_alloc_extended_subtable() is called. + * + * Returns 0 if successful, or a non-zero error otherwise. + */ +static int gasket_alloc_extended_subtable( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, + u64 __iomem *slot) +{ + ulong page_addr, subtable_bytes; + dma_addr_t dma_addr; + + /* XXX FIX ME XXX this is inefficient for non-4K page sizes */ + + /* GFP_DMA flag must be passed to architectures for which + * part of the memory range is not considered DMA'able. + * This seems to be the case for Juno board with 4.5.0 Linaro kernel + */ + page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA); + if (!page_addr) + return -ENOMEM; + pte->page = virt_to_page((void *)page_addr); + pte->offset = 0; + + subtable_bytes = sizeof(struct gasket_page_table_entry) * + GASKET_PAGES_PER_SUBTABLE; + pte->sublevel = vmalloc(subtable_bytes); + if (!pte->sublevel) { + free_page(page_addr); + memset(pte, 0, sizeof(struct gasket_page_table_entry)); + return -ENOMEM; + } + memset(pte->sublevel, 0, subtable_bytes); + + /* Map the page into DMA space. */ + if (pg_tbl->dma_ops) { + pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, + PAGE_SIZE, DMA_BIDIRECTIONAL); + } else { + pte->dma_addr = _no_op_dma_map_page(pg_tbl->device, pte->page, + 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + } + /* Wait until the page is mapped. */ + mb(); + + /* make the addresses available to the device */ + dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; + writeq(dma_addr, slot); + + pte->status = PTE_INUSE; + + return 0; +} + +/* Unmapping functions */ +/* + * Non-locking entry to unmapping routines. + * @pg_tbl: Gasket page table structure. + * @dev_addr: Starting device address of the pages to unmap. + * @num_pages: The number of device pages to unmap. + * + * Description: Version of gasket_unmap_pages that assumes the page table lock + * is held. + */ +static void gasket_page_table_unmap_nolock( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + if (!num_pages) + return; + + if (gasket_addr_is_simple(pg_tbl, dev_addr)) + gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages); + else + gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages); +} + +/* + * Unmap and release pages mapped to simple addresses. + * @pg_tbl: Gasket page table pointer. + * @dev_addr: Starting device address of the buffers. + * @num_pages: The number of device pages to unmap. + * + * Description: gasket_simple_unmap_pages calls gasket_perform_unmapping() to + * unmap and release the buffers in the level 0 page table. + * + * The sd_mutex must be held when gasket_unmap_simple_pages() is called. + */ +static void gasket_unmap_simple_pages( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); + + gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot, + pg_tbl->base_slot + slot, num_pages, 1); +} + +/** + * Unmap and release buffers to extended addresses. + * @pg_tbl: Gasket page table pointer. + * @dev_addr: Starting device address of the pages to unmap. + * @addr: Starting device address of the buffers. + * @num_pages: The number of device pages to unmap. + * + * Description: gasket_extended_unmap_pages loops over the level 0 page table + * entries, and for each calls gasket_perform_unmapping() to unmap + * the buffers from the level 1 page [sub]table for that level 0 + * entry. + * + * The page table mutex must be held when + * gasket_unmap_extended_pages() is called. + */ +static void gasket_unmap_extended_pages( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + uint slot_idx, remain, len; + struct gasket_page_table_entry *pte; + u64 __iomem *slot_base; + + remain = num_pages; + slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pg_tbl->entries + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + while (remain > 0) { + /* TODO: Add check to ensure pte remains valid? */ + len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); + + if (pte->status == PTE_INUSE) { + slot_base = (u64 __iomem *)(page_address(pte->page) + + pte->offset); + gasket_perform_unmapping( + pg_tbl, pte->sublevel + slot_idx, + slot_base + slot_idx, len, 0); + } + + remain -= len; + slot_idx = 0; + pte++; + } +} + +/* + * Unmap and release mapped pages. + * @pg_tbl: Gasket page table pointer. + * @ptes: Array of page table entries to describe the mapped range, one per + * page to unmap. + * @slots: Device slots corresponding to the mappings described by "ptes". + * As with ptes, one element per page to unmap. + * If these are simple mappings, these will be address translation + * registers. If these are extended mappings, these will be witin a + * second-level page table allocated on the host, and so must have + * their __iomem attribute casted away. + * @num_pages: Number of pages to unmap. + * @is_simple_mapping: 1 if this is a simple mapping, 0 otherwise. + * + * Description: gasket_perform_unmapping() loops through the metadata entries + * in a last level page table (simple table or extended subtable), + * and for each page: + * - Unmaps the page from DMA space (dma_unmap_page), + * - Returns the page to the OS (gasket_release_page), + * The entry in the page table is written to 0. The metadata + * type is set to PTE_FREE and the metadata is all reset + * to 0. + * + * The page table mutex must be held when this function is called. + */ +static void gasket_perform_unmapping( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, + u64 __iomem *slots, uint num_pages, int is_simple_mapping) +{ + int i; + /* + * For each page table entry and corresponding entry in the device's + * address translation table: + */ + for (i = 0; i < num_pages; i++) { + /* release the address from the device, */ + if (is_simple_mapping || ptes[i].status == PTE_INUSE) + writeq(0, &slots[i]); + else + ((u64 __force *)slots)[i] = 0; + /* Force sync around the address release. */ + mb(); + + /* release the address from the driver, */ + if (ptes[i].status == PTE_INUSE) { + if (ptes[i].dma_addr) { + dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, + PAGE_SIZE, DMA_FROM_DEVICE); + } + if (gasket_release_page(ptes[i].page)) + --pg_tbl->num_active_pages; + } + ptes[i].status = PTE_FREE; + + /* and clear the PTE. */ + memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); + } +} + +/* + * Free a second level page [sub]table. + * @pg_tbl: Gasket page table pointer. + * @pte: Page table entry _pointing_to_ the subtable to free. + * @slot: Device slot holding a pointer to the sublevel's contents. + * + * Description: Safely deallocates a second-level [sub]table by: + * - Marking the containing first-level PTE as free + * - Setting the corresponding [extended] device slot as NULL + * - Unmapping the PTE from DMA space. + * - Freeing the subtable's memory. + * - Deallocating the page and clearing out the PTE. + * + * The page table mutex must be held before this call. + */ +static void gasket_free_extended_subtable( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, + u64 __iomem *slot) +{ + /* Release the page table from the driver */ + pte->status = PTE_FREE; + + /* Release the page table from the device */ + writeq(0, slot); + /* Force sync around the address release. */ + mb(); + + if (pte->dma_addr) + dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + vfree(pte->sublevel); + + if (pte->page) + free_page((ulong)page_address(pte->page)); + + memset(pte, 0, sizeof(struct gasket_page_table_entry)); +} + +/* + * Safely return a page to the OS. + * @page: The page to return to the OS. + * Returns 1 if the page was released, 0 if it was + * ignored. + */ +static int gasket_release_page(struct page *page) +{ + if (!page) + return 0; + + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); + + return 1; +} + +/* Evaluates to nonzero if the specified virtual address is simple. */ +static inline int gasket_addr_is_simple( + struct gasket_page_table *pg_tbl, ulong addr) +{ + return !((addr) & (pg_tbl)->extended_flag); +} + +/* + * Validity checking for simple addresses. + * @pg_tbl: Gasket page table pointer. + * @dev_addr: The device address to which the pages will be mapped. + * @num_pages: The number of pages in the range to consider. + * + * Description: This call verifies that address translation commutes (from + * address to/from page + offset) and that the requested page range starts and + * ends within the set of currently-partitioned simple pages. + */ +static int gasket_is_simple_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + ulong page_offset = dev_addr & (PAGE_SIZE - 1); + ulong page_index = + (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1); + + if (gasket_components_to_dev_address( + pg_tbl, 1, page_index, page_offset) != dev_addr) { + gasket_pg_tbl_error( + pg_tbl, "address is invalid, 0x%lX", dev_addr); + return 1; + } + + if (page_index >= pg_tbl->num_simple_entries) { + gasket_pg_tbl_error( + pg_tbl, + "starting slot at %lu is too large, max is < %u", + page_index, pg_tbl->num_simple_entries); + return 1; + } + + if (page_index + num_pages > pg_tbl->num_simple_entries) { + gasket_pg_tbl_error( + pg_tbl, + "ending slot at %lu is too large, max is <= %u", + page_index + num_pages, pg_tbl->num_simple_entries); + return 1; + } + + return 0; +} + +/* + * Verifies that address translation commutes (from address to/from page + + * offset) and that the requested page range starts and ends within the set of + * currently-partitioned simple pages. + * + * @pg_tbl: Gasket page table pointer. + * @dev_addr: The device address to which the pages will be mapped. + * @num_pages: The number of second-level/sub pages in the range to consider. + */ +static int gasket_is_extended_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + /* Starting byte index of dev_addr into the first mapped page */ + ulong page_offset = dev_addr & (PAGE_SIZE - 1); + ulong page_global_idx, page_lvl0_idx; + ulong num_lvl0_pages; + ulong addr; + + /* check if the device address is out of bound */ + addr = dev_addr & ~((pg_tbl)->extended_flag); + if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { + gasket_pg_tbl_error(pg_tbl, "device address out of bound, 0x%p", + (void *)dev_addr); + return 1; + } + + /* Find the starting sub-page index in the space of all sub-pages. */ + page_global_idx = (dev_addr / PAGE_SIZE) & + (pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1); + + /* Find the starting level 0 index. */ + page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + /* Get the count of affected level 0 pages. */ + num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) / + GASKET_PAGES_PER_SUBTABLE; + + if (gasket_components_to_dev_address( + pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { + gasket_pg_tbl_error( + pg_tbl, "address is invalid, 0x%p", (void *)dev_addr); + return 1; + } + + if (page_lvl0_idx >= pg_tbl->num_extended_entries) { + gasket_pg_tbl_error( + pg_tbl, + "starting level 0 slot at %lu is too large, max is < " + "%u", page_lvl0_idx, pg_tbl->num_extended_entries); + return 1; + } + + if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) { + gasket_pg_tbl_error( + pg_tbl, + "ending level 0 slot at %lu is too large, max is <= %u", + page_lvl0_idx + num_lvl0_pages, + pg_tbl->num_extended_entries); + return 1; + } + + return 0; +} + +/* + * Checks if a range of PTEs is free. + * @ptes: The set of PTEs to check. + * @num_entries: The number of PTEs to check. + * + * Description: Iterates over the input PTEs to determine if all have been + * marked as FREE or if any are INUSE. In the former case, 1/true is returned. + * Otherwise, 0/false is returned. + * + * The page table mutex must be held before this call. + */ +static int gasket_is_pte_range_free( + struct gasket_page_table_entry *ptes, uint num_entries) +{ + int i; + + for (i = 0; i < num_entries; i++) { + if (ptes[i].status != PTE_FREE) + return 0; + } + + return 1; +} + +/* + * Actually perform collection. + * @pg_tbl: Gasket page table structure. + * + * Description: Version of gasket_page_table_garbage_collect that assumes the + * page table lock is held. + */ +static void gasket_page_table_garbage_collect_nolock( + struct gasket_page_table *pg_tbl) +{ + struct gasket_page_table_entry *pte; + u64 __iomem *slot; + + /* XXX FIX ME XXX -- more efficient to keep a usage count */ + /* rather than scanning the second level page tables */ + + for (pte = pg_tbl->entries + pg_tbl->num_simple_entries, + slot = pg_tbl->base_slot + pg_tbl->num_simple_entries; + pte < pg_tbl->entries + pg_tbl->config.total_entries; + pte++, slot++) { + if (pte->status == PTE_INUSE) { + if (gasket_is_pte_range_free( + pte->sublevel, GASKET_PAGES_PER_SUBTABLE)) + gasket_free_extended_subtable( + pg_tbl, pte, slot); + } + } +} + +/* + * Converts components to a device address. + * @pg_tbl: Gasket page table structure. + * @is_simple: nonzero if this should be a simple entry, zero otherwise. + * @page_index: The page index into the respective table. + * @offset: The offset within the requested page. + * + * Simple utility function to convert (simple, page, offset) into a device + * address. + * Examples: + * Simple page 0, offset 32: + * Input (0, 0, 32), Output 0x20 + * Simple page 1000, offset 511: + * Input (0, 1000, 512), Output 0x3E81FF + * Extended page 0, offset 32: + * Input (0, 0, 32), Output 0x8000000020 + * Extended page 1000, offset 511: + * Input (1, 1000, 512), Output 0x8003E81FF + */ +static ulong gasket_components_to_dev_address( + struct gasket_page_table *pg_tbl, int is_simple, uint page_index, + uint offset) +{ + ulong lvl0_index, lvl1_index; + + if (is_simple) { + /* Return simple addresses directly. */ + lvl0_index = page_index & (pg_tbl->config.total_entries - 1); + return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; + } + + /* + * This could be compressed into fewer statements, but + * A) the compiler should optimize it + * B) this is not slow + * C) this is an uncommon operation + * D) this is actually readable this way. + */ + lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE; + lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1); + return (pg_tbl)->extended_flag | + (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) | + (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset; +} + +/* + * Gets the index of the address' page in the simple table. + * @pg_tbl: Gasket page table structure. + * @dev_addr: The address whose page index to retrieve. + * + * Description: Treats the input address as a simple address and determines the + * index of its underlying page in the simple page table (i.e., device address + * translation registers. + * + * Does not perform validity checking. + */ +static int gasket_simple_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr) +{ + return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & + (pg_tbl->config.total_entries - 1); +} + +/* + * Gets the level 0 page index for the given address. + * @pg_tbl: Gasket page table structure. + * @dev_addr: The address whose page index to retrieve. + * + * Description: Treats the input address as an extended address and determines + * the index of its underlying page in the first-level extended page table + * (i.e., device extended address translation registers). + * + * Does not perform validity checking. + */ +static ulong gasket_extended_lvl0_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr) +{ + return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & + ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); +} + +/* + * Gets the level 1 page index for the given address. + * @pg_tbl: Gasket page table structure. + * @dev_addr: The address whose page index to retrieve. + * + * Description: Treats the input address as an extended address and determines + * the index of its underlying page in the second-level extended page table + * (i.e., host memory pointed to by a first-level page table entry). + * + * Does not perform validity checking. + */ +static ulong gasket_extended_lvl1_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr) +{ + return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & + (GASKET_PAGES_PER_SUBTABLE - 1); +} + +/* + * Determines whether a host buffer was mapped as coherent memory. + * @pg_tbl: gasket_page_table structure tracking the host buffer mapping + * @host_addr: user virtual address within a host buffer + * + * Description: A Gasket page_table currently support one contiguous + * dma range, mapped to one contiguous virtual memory range. Check if the + * host_addr is within start of page 0, and end of last page, for that range. + */ +static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) +{ + u64 min, max; + + /* whether the host address is within user virt range */ + if (!pg_tbl->coherent_pages) + return 0; + + min = (u64)pg_tbl->coherent_pages[0].user_virt; + max = min + PAGE_SIZE * pg_tbl->num_coherent_pages; + + return min <= host_addr && host_addr < max; +} + +/* + * Records the host_addr to coherent dma memory mapping. + * @gasket_dev: Gasket Device. + * @size: Size of the virtual address range to map. + * @dma_address: Dma address within the coherent memory range. + * @vma: Virtual address we wish to map to coherent memory. + * + * Description: For each page in the virtual address range, record the + * coherent page mgasket_pretapping. + */ +int gasket_set_user_virt( + struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address, + ulong vma) +{ + int j; + struct gasket_page_table *pg_tbl; + + unsigned int num_pages = size / PAGE_SIZE; + + /* + * TODO: for future chipset, better handling of the case where multiple + * page tables are supported on a given device + */ + pg_tbl = gasket_dev->page_table[0]; + if (!pg_tbl) { + gasket_nodev_error( + "gasket_set_user_virt: invalid page table index"); + return 0; + } + for (j = 0; j < num_pages; j++) { + pg_tbl->coherent_pages[j].user_virt = + (u64)vma + j * PAGE_SIZE; + } + return 0; +} + +/* + * Allocate a block of coherent memory. + * @gasket_dev: Gasket Device. + * @size: Size of the memory block. + * @dma_address: Dma address allocated by the kernel. + * @index: Index of the gasket_page_table within this Gasket device + * + * Description: Allocate a contiguous coherent memory block, DMA'ble + * by this device. + */ +int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, + dma_addr_t *dma_address, u64 index) +{ + dma_addr_t handle; + void *mem; + int j; + unsigned int num_pages = (size + PAGE_SIZE - 1) / (PAGE_SIZE); + const struct gasket_driver_desc *driver_desc = + gasket_get_driver_desc(gasket_dev); + + if (!gasket_dev->page_table[index]) + return -EFAULT; + + if (num_pages == 0) + return -EINVAL; + + mem = dma_alloc_coherent(gasket_get_device(gasket_dev), + num_pages * PAGE_SIZE, &handle, 0); + if (!mem) + goto nomem; + + gasket_dev->page_table[index]->num_coherent_pages = num_pages; + + /* allocate the physical memory block */ + gasket_dev->page_table[index]->coherent_pages = kzalloc( + num_pages * sizeof(struct gasket_coherent_page_entry), + GFP_KERNEL); + if (!gasket_dev->page_table[index]->coherent_pages) + goto nomem; + *dma_address = 0; + + gasket_dev->coherent_buffer.length_bytes = + PAGE_SIZE * (num_pages); + gasket_dev->coherent_buffer.phys_base = handle; + gasket_dev->coherent_buffer.virt_base = mem; + + *dma_address = driver_desc->coherent_buffer_description.base; + for (j = 0; j < num_pages; j++) { + gasket_dev->page_table[index]->coherent_pages[j].paddr = + handle + j * PAGE_SIZE; + gasket_dev->page_table[index]->coherent_pages[j].kernel_virt = + (u64)mem + j * PAGE_SIZE; + } + + if (*dma_address == 0) + goto nomem; + return 0; + +nomem: + if (mem) { + dma_free_coherent(gasket_get_device(gasket_dev), + num_pages * PAGE_SIZE, mem, handle); + } + + if (gasket_dev->page_table[index]->coherent_pages) { + kfree(gasket_dev->page_table[index]->coherent_pages); + gasket_dev->page_table[index]->coherent_pages = 0; + } + gasket_dev->page_table[index]->num_coherent_pages = 0; + return -ENOMEM; +} + +/* + * Free a block of coherent memory. + * @gasket_dev: Gasket Device. + * @size: Size of the memory block. + * @dma_address: Dma address allocated by the kernel. + * @index: Index of the gasket_page_table within this Gasket device + * + * Description: Release memory allocated thru gasket_alloc_coherent_memory. + */ +int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, + dma_addr_t dma_address, u64 index) +{ + const struct gasket_driver_desc *driver_desc; + + if (!gasket_dev->page_table[index]) + return -EFAULT; + + driver_desc = gasket_get_driver_desc(gasket_dev); + + if (driver_desc->coherent_buffer_description.base != dma_address) + return -EADDRNOTAVAIL; + + if (gasket_dev->coherent_buffer.length_bytes) { + dma_free_coherent(gasket_get_device(gasket_dev), + gasket_dev->coherent_buffer.length_bytes, + gasket_dev->coherent_buffer.virt_base, + gasket_dev->coherent_buffer.phys_base); + gasket_dev->coherent_buffer.length_bytes = 0; + gasket_dev->coherent_buffer.virt_base = 0; + gasket_dev->coherent_buffer.phys_base = 0; + } + return 0; +} + +/* + * Release all coherent memory. + * @gasket_dev: Gasket Device. + * @index: Index of the gasket_page_table within this Gasket device + * + * Description: Release all memory allocated thru gasket_alloc_coherent_memory. + */ +void gasket_free_coherent_memory_all( + struct gasket_dev *gasket_dev, u64 index) +{ + if (!gasket_dev->page_table[index]) + return; + + if (gasket_dev->coherent_buffer.length_bytes) { + dma_free_coherent(gasket_get_device(gasket_dev), + gasket_dev->coherent_buffer.length_bytes, + gasket_dev->coherent_buffer.virt_base, + gasket_dev->coherent_buffer.phys_base); + gasket_dev->coherent_buffer.length_bytes = 0; + gasket_dev->coherent_buffer.virt_base = 0; + gasket_dev->coherent_buffer.phys_base = 0; + } +} diff --git a/gasket_page_table.h b/gasket_page_table.h new file mode 100644 index 0000000..f2f519a --- /dev/null +++ b/gasket_page_table.h @@ -0,0 +1,265 @@ +/* Gasket Page Table functionality. This file describes the address + * translation/paging functionality supported by the Gasket driver framework. + * As much as possible, internal details are hidden to simplify use - + * all calls are thread-safe (protected by an internal mutex) except where + * indicated otherwise. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __GASKET_ADDR_TRNSL_H__ +#define __GASKET_ADDR_TRNSL_H__ + +#include +#include + +#include "gasket_constants.h" +#include "gasket_core.h" + +/* + * Structure used for managing address translation on a device. All details are + * internal to the implementation. + */ +struct gasket_page_table; + +/* + * Allocate and init address translation data. + * @ppage_table: Pointer to Gasket page table pointer. Set by this call. + * @att_base_reg: [Mapped] pointer to the first entry in the device's address + * translation table. + * @extended_offset_reg: [Mapped] pointer to the device's register containing + * the starting index of the extended translation table. + * @extended_bit_location: The index of the bit indicating whether an address + * is extended. + * @total_entries: The total number of entries in the device's address + * translation table. + * @device: Device structure for the underlying device. Only used for logging. + * @pci_dev: PCI system descriptor for the underlying device. + * @bool has_dma_ops: Whether the page table uses arch specific dma_ops or + * whether the driver will supply its own. + * + * Description: Allocates and initializes data to track address translation - + * simple and extended page table metadata. Initially, the page table is + * partitioned such that all addresses are "simple" (single-level lookup). + * gasket_partition_page_table can be called to change this paritioning. + * + * Returns 0 on success, a negative error code otherwise. + */ +int gasket_page_table_init( + struct gasket_page_table **ppg_tbl, + const struct gasket_bar_data *bar_data, + const struct gasket_page_table_config *page_table_config, + struct device *device, struct pci_dev *pci_dev, bool dma_ops); + +/* + * Deallocate and cleanup page table data. + * @page_table: Gasket page table pointer. + * + * Description: The inverse of gasket_init; frees page_table and its contained + * elements. + * + * Because this call destroys the page table, it cannot be + * thread-safe (mutex-protected)! + */ +void gasket_page_table_cleanup(struct gasket_page_table *page_table); + +/* + * Sets the size of the simple page table. + * @page_table: Gasket page table pointer. + * @num_simple_entries: Desired size of the simple page table (in entries). + * + * Description: gasket_partition_page_table checks to see if the simple page + * size can be changed (i.e., if there are no active extended + * mappings in the new simple size range), and, if so, + * sets the new simple and extended page table sizes. + * + * Returns 0 if successful, or non-zero if the page table entries + * are not free. + */ +int gasket_page_table_partition( + struct gasket_page_table *page_table, uint num_simple_entries); + +/* + * Get and map [host] user space pages into device memory. + * @page_table: Gasket page table pointer. + * @host_addr: Starting host virtual memory address of the pages. + * @dev_addr: Starting device address of the pages. + * @num_pages: Number of [4kB] pages to map. + * + * Description: Maps the "num_pages" pages of host memory pointed to by + * host_addr to the address "dev_addr" in device memory. + * + * The caller is responsible for checking the addresses ranges. + * + * Returns 0 if successful or a non-zero error number otherwise. + * If there is an error, no pages are mapped. + */ +int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, + ulong dev_addr, uint num_pages); + +/* + * Un-map host pages from device memory. + * @page_table: Gasket page table pointer. + * @dev_addr: Starting device address of the pages to unmap. + * @num_pages: The number of [4kB] pages to unmap. + * + * Description: The inverse of gasket_map_pages. Unmaps pages from the device. + */ +void gasket_page_table_unmap( + struct gasket_page_table *page_table, ulong dev_addr, uint num_pages); + +/* + * Unmap ALL host pages from device memory. + * @page_table: Gasket page table pointer. + */ +void gasket_page_table_unmap_all(struct gasket_page_table *page_table); + +/* + * Unmap all host pages from device memory and reset the table to fully simple + * addressing. + * @page_table: Gasket page table pointer. + */ +void gasket_page_table_reset(struct gasket_page_table *page_table); + +/* + * Reclaims unused page table memory. + * @page_table: Gasket page table pointer. + * + * Description: Examines the page table and frees any currently-unused + * allocations. Called internally on gasket_cleanup(). + */ +void gasket_page_table_garbage_collect(struct gasket_page_table *page_table); + +/* + * Retrieve the backing page for a device address. + * @page_table: Gasket page table pointer. + * @dev_addr: Gasket device address. + * @ppage: Pointer to a page pointer for the returned page. + * @poffset: Pointer to an unsigned long for the returned offset. + * + * Description: Interprets the address and looks up the corresponding page + * in the page table and the offset in that page. (We need an + * offset because the host page may be larger than the Gasket chip + * page it contains.) + * + * Returns 0 if successful, -1 for an error. The page pointer + * and offset are returned through the pointers, if successful. + */ +int gasket_page_table_lookup_page( + struct gasket_page_table *page_table, ulong dev_addr, + struct page **page, ulong *poffset); + +/* + * Checks validity for input addrs and size. + * @page_table: Gasket page table pointer. + * @host_addr: Host address to check. + * @dev_addr: Gasket device address. + * @bytes: Size of the range to check (in bytes). + * + * Description: This call performs a number of checks to verify that the ranges + * specified by both addresses and the size are valid for mapping pages into + * device memory. + * + * Returns 1 if true - if the mapping is bad, 0 otherwise. + */ +int gasket_page_table_are_addrs_bad( + struct gasket_page_table *page_table, ulong host_addr, ulong dev_addr, + ulong bytes); + +/* + * Checks validity for input dev addr and size. + * @page_table: Gasket page table pointer. + * @dev_addr: Gasket device address. + * @bytes: Size of the range to check (in bytes). + * + * Description: This call performs a number of checks to verify that the range + * specified by the device address and the size is valid for mapping pages into + * device memory. + * + * Returns 1 if true - if the address is bad, 0 otherwise. + */ +int gasket_page_table_is_dev_addr_bad( + struct gasket_page_table *page_table, ulong dev_addr, ulong bytes); + +/* + * Gets maximum size for the given page table. + * @page_table: Gasket page table pointer. + */ +uint gasket_page_table_max_size(struct gasket_page_table *page_table); + +/* + * Gets the total number of entries in the arg. + * @page_table: Gasket page table pointer. + */ +uint gasket_page_table_num_entries(struct gasket_page_table *page_table); + +/* + * Gets the number of simple entries. + * @page_table: Gasket page table pointer. + */ +uint gasket_page_table_num_simple_entries(struct gasket_page_table *page_table); + +/* + * Gets the number of extended entries. + * @page_table: Gasket page table pointer. + */ +uint gasket_page_table_num_extended_entries( + struct gasket_page_table *page_table); + +/* + * Gets the number of actively pinned pages. + * @page_table: Gasket page table pointer. + */ +uint gasket_page_table_num_active_pages(struct gasket_page_table *page_table); + +/* + * Get status of page table managed by @page_table. + * @page_table: Gasket page table pointer. + */ +int gasket_page_table_system_status(struct gasket_page_table *page_table); + +/* + * Allocate a block of coherent memory. + * @gasket_dev: Gasket Device. + * @size: Size of the memory block. + * @dma_address: Dma address allocated by the kernel. + * @index: Index of the gasket_page_table within this Gasket device + * + * Description: Allocate a contiguous coherent memory block, DMA'ble + * by this device. + */ +int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, uint64_t size, + dma_addr_t *dma_address, uint64_t index); +/* Release a block of contiguous coherent memory, in use by a device. */ +int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, uint64_t size, + dma_addr_t dma_address, uint64_t index); + +/* Release all coherent memory. */ +void gasket_free_coherent_memory_all(struct gasket_dev *gasket_dev, + uint64_t index); + +/* + * Records the host_addr to coherent dma memory mapping. + * @gasket_dev: Gasket Device. + * @size: Size of the virtual address range to map. + * @dma_address: Dma address within the coherent memory range. + * @vma: Virtual address we wish to map to coherent memory. + * + * Description: For each page in the virtual address range, record the + * coherent page mapping. + * + * Does not perform validity checking. + */ +int gasket_set_user_virt(struct gasket_dev *gasket_dev, uint64_t size, + dma_addr_t dma_address, ulong vma); + +#endif diff --git a/gasket_sysfs.c b/gasket_sysfs.c new file mode 100644 index 0000000..d45098c --- /dev/null +++ b/gasket_sysfs.c @@ -0,0 +1,497 @@ +/* Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include "gasket_sysfs.h" + +#include "gasket_core.h" +#include "gasket_logging.h" + +/* + * Pair of kernel device and user-specified pointer. Used in lookups in sysfs + * "show" functions to return user data. + */ + +struct gasket_sysfs_mapping { + /* + * The device bound to this mapping. If this is NULL, then this mapping + * is free. + */ + struct device *device; + + /* Legacy device struct, if used by this mapping's driver. */ + struct device *legacy_device; + + /* The Gasket descriptor for this device. */ + struct gasket_dev *gasket_dev; + + /* This device's set of sysfs attributes/nodes. */ + struct gasket_sysfs_attribute *attributes; + + /* The number of live elements in "attributes". */ + int attribute_count; + + /* Protects structure from simultaneous access. */ + struct mutex mutex; + + /* Tracks active users of this mapping. */ + struct kref refcount; +}; + +/* + * Data needed to manage users of this sysfs utility. + * Currently has a fixed size; if space is a concern, this can be dynamically + * allocated. + */ +/* + * 'Global' (file-scoped) list of mappings between devices and gasket_data + * pointers. This removes the requirement to have a gasket_sysfs_data + * handle in all files. + */ +static struct gasket_sysfs_mapping dev_mappings[GASKET_SYSFS_NUM_MAPPINGS]; + +/* + * Callback when a mapping's refcount goes to zero. + * @ref: The reference count of the containing sysfs mapping. + */ +static void release_entry(struct kref *ref) +{ + /* All work is done after the return from kref_put. */ +} + +/* + * Looks up mapping information for the given device. + * @device: The device whose mapping to look for. + * + * Looks up the requested device and takes a reference and returns it if found, + * and returns NULL otherwise. + */ +static struct gasket_sysfs_mapping *get_mapping(struct device *device) +{ + int i; + + if (!device) { + gasket_nodev_error("Received NULL device!"); + return NULL; + } + + for (i = 0; i < GASKET_SYSFS_NUM_MAPPINGS; i++) { + mutex_lock(&dev_mappings[i].mutex); + if (dev_mappings[i].device == device || + dev_mappings[i].legacy_device == device) { + kref_get(&dev_mappings[i].refcount); + mutex_unlock(&dev_mappings[i].mutex); + return &dev_mappings[i]; + } + mutex_unlock(&dev_mappings[i].mutex); + } + + gasket_nodev_info("Mapping to device %s not found.", device->kobj.name); + return NULL; +} + +/* + * Returns a reference to a mapping. + * @mapping: The mapping we're returning. + * + * Decrements the refcount for the given mapping (if valid). If the refcount is + * zero, then it cleans up the mapping - in this function as opposed to the + * kref_put callback, due to a potential deadlock. + * + * Although put_mapping_n exists, this function is left here (as an implicit + * put_mapping_n(..., 1) for convenience. + */ +static void put_mapping(struct gasket_sysfs_mapping *mapping) +{ + int i; + int num_files_to_remove = 0; + struct device_attribute *files_to_remove; + struct device *device; + struct device *legacy_device; + + if (!mapping) { + gasket_nodev_info("Mapping should not be NULL."); + return; + } + + mutex_lock(&mapping->mutex); + if (mapping->refcount.refcount.refs.counter == 0) + gasket_nodev_error("Refcount is already 0!"); + if (kref_put(&mapping->refcount, release_entry)) { + gasket_nodev_info("Removing Gasket sysfs mapping, device %s", + mapping->device->kobj.name); + /* + * We can't remove the sysfs nodes in the kref callback, since + * device_remove_file() blocks until the node is free. + * Readers/writers of sysfs nodes, though, will be blocked on + * the mapping mutex, resulting in deadlock. To fix this, the + * sysfs nodes are removed outside the lock. + */ + device = mapping->device; + legacy_device = mapping->legacy_device; + num_files_to_remove = mapping->attribute_count; + files_to_remove = kzalloc( + num_files_to_remove * sizeof(*files_to_remove), + GFP_KERNEL); + for (i = 0; i < num_files_to_remove; i++) + files_to_remove[i] = mapping->attributes[i].attr; + + kfree(mapping->attributes); + mapping->attributes = NULL; + mapping->attribute_count = 0; + mapping->device = NULL; + mapping->gasket_dev = NULL; + } + mutex_unlock(&mapping->mutex); + + if (num_files_to_remove != 0) { + for (i = 0; i < num_files_to_remove; ++i) { + device_remove_file(device, &files_to_remove[i]); + if (legacy_device) + device_remove_file( + legacy_device, &files_to_remove[i]); + } + kfree(files_to_remove); + } +} + +/* + * Returns a reference N times. + * @mapping: The mapping to return. + * + * In higher-level resource acquire/release function pairs, the release function + * will need to release a mapping 2x - once for the refcount taken in the + * release function itself, and once for the count taken in the acquire call. + */ +static void put_mapping_n(struct gasket_sysfs_mapping *mapping, int times) +{ + int i; + + for (i = 0; i < times; i++) + put_mapping(mapping); +} + +void gasket_sysfs_init(void) +{ + int i; + + for (i = 0; i < GASKET_SYSFS_NUM_MAPPINGS; i++) { + dev_mappings[i].device = NULL; + mutex_init(&dev_mappings[i].mutex); + } +} + +int gasket_sysfs_create_mapping( + struct device *device, struct gasket_dev *gasket_dev) +{ + struct gasket_sysfs_mapping *mapping; + int map_idx = -1; + + /* + * We need a function-level mutex to protect against the same device + * being added [multiple times] simultaneously. + */ + static DEFINE_MUTEX(function_mutex); + + mutex_lock(&function_mutex); + + gasket_nodev_info( + "Creating sysfs entries for device pointer 0x%p.", device); + + /* Check that the device we're adding hasn't already been added. */ + mapping = get_mapping(device); + if (mapping) { + gasket_nodev_error( + "Attempting to re-initialize sysfs mapping for device " + "0x%p.", device); + put_mapping(mapping); + mutex_unlock(&function_mutex); + return -EINVAL; + } + + /* Find the first empty entry in the array. */ + for (map_idx = 0; map_idx < GASKET_SYSFS_NUM_MAPPINGS; ++map_idx) { + mutex_lock(&dev_mappings[map_idx].mutex); + if (!dev_mappings[map_idx].device) + /* Break with the mutex held! */ + break; + mutex_unlock(&dev_mappings[map_idx].mutex); + } + + if (map_idx == GASKET_SYSFS_NUM_MAPPINGS) { + gasket_nodev_error("All mappings have been exhausted!"); + mutex_unlock(&function_mutex); + return -ENOMEM; + } + + gasket_nodev_info( + "Creating sysfs mapping for device %s.", device->kobj.name); + + mapping = &dev_mappings[map_idx]; + kref_init(&mapping->refcount); + mapping->device = device; + mapping->gasket_dev = gasket_dev; + mapping->attributes = kzalloc( + GASKET_SYSFS_MAX_NODES * sizeof(*mapping->attributes), + GFP_KERNEL); + mapping->attribute_count = 0; + if (!mapping->attributes) { + gasket_nodev_error("Unable to allocate sysfs attribute array."); + mutex_unlock(&mapping->mutex); + mutex_unlock(&function_mutex); + return -ENOMEM; + } + + mutex_unlock(&mapping->mutex); + mutex_unlock(&function_mutex); + + /* Don't decrement the refcount here! One open count keeps it alive! */ + return 0; +} + +int gasket_sysfs_create_entries( + struct device *device, const struct gasket_sysfs_attribute *attrs) +{ + int i; + int ret; + struct gasket_sysfs_mapping *mapping = get_mapping(device); + + if (!mapping) { + gasket_nodev_error( + "Creating entries for device 0x%p without first " + "initializing mapping.", + device); + return -EINVAL; + } + + mutex_lock(&mapping->mutex); + for (i = 0; strcmp(attrs[i].attr.attr.name, GASKET_ARRAY_END_MARKER); + i++) { + if (mapping->attribute_count == GASKET_SYSFS_MAX_NODES) { + gasket_nodev_error( + "Maximum number of sysfs nodes reached for " + "device."); + mutex_unlock(&mapping->mutex); + put_mapping(mapping); + return -ENOMEM; + } + + ret = device_create_file(device, &attrs[i].attr); + if (ret) { + gasket_nodev_error("Unable to create device entries"); + mutex_unlock(&mapping->mutex); + put_mapping(mapping); + return ret; + } + + if (mapping->legacy_device) { + ret = device_create_file(mapping->legacy_device, + &attrs[i].attr); + if (ret) { + gasket_log_error( + mapping->gasket_dev, + "Unable to create legacy sysfs entries;" + " rc: %d", + ret); + mutex_unlock(&mapping->mutex); + put_mapping(mapping); + return ret; + } + } + + mapping->attributes[mapping->attribute_count] = attrs[i]; + ++mapping->attribute_count; + } + + mutex_unlock(&mapping->mutex); + put_mapping(mapping); + return 0; +} +EXPORT_SYMBOL(gasket_sysfs_create_entries); + +void gasket_sysfs_remove_mapping(struct device *device) +{ + struct gasket_sysfs_mapping *mapping = get_mapping(device); + + if (!mapping) { + gasket_nodev_error( + "Attempted to remove non-existent sysfs mapping to " + "device 0x%p", + device); + return; + } + + put_mapping_n(mapping, 2); +} + +struct gasket_dev *gasket_sysfs_get_device_data(struct device *device) +{ + struct gasket_sysfs_mapping *mapping = get_mapping(device); + + if (!mapping) { + gasket_nodev_error("device %p not registered.", device); + return NULL; + } + + return mapping->gasket_dev; +} +EXPORT_SYMBOL(gasket_sysfs_get_device_data); + +void gasket_sysfs_put_device_data(struct device *device, struct gasket_dev *dev) +{ + struct gasket_sysfs_mapping *mapping = get_mapping(device); + + if (!mapping) + return; + + /* See comment of put_mapping_n() for why the '2' is necessary. */ + put_mapping_n(mapping, 2); +} +EXPORT_SYMBOL(gasket_sysfs_put_device_data); + +struct gasket_sysfs_attribute *gasket_sysfs_get_attr( + struct device *device, struct device_attribute *attr) +{ + int i; + int num_attrs; + struct gasket_sysfs_mapping *mapping = get_mapping(device); + struct gasket_sysfs_attribute *attrs = NULL; + + if (!mapping) + return NULL; + + attrs = mapping->attributes; + num_attrs = mapping->attribute_count; + for (i = 0; i < num_attrs; ++i) { + if (!strcmp(attrs[i].attr.attr.name, attr->attr.name)) + return &attrs[i]; + } + + gasket_nodev_error("Unable to find match for device_attribute %s", + attr->attr.name); + return NULL; +} +EXPORT_SYMBOL(gasket_sysfs_get_attr); + +void gasket_sysfs_put_attr( + struct device *device, struct gasket_sysfs_attribute *attr) +{ + int i; + int num_attrs; + struct gasket_sysfs_mapping *mapping = get_mapping(device); + struct gasket_sysfs_attribute *attrs = NULL; + + if (!mapping) + return; + + attrs = mapping->attributes; + num_attrs = mapping->attribute_count; + for (i = 0; i < num_attrs; ++i) { + if (&attrs[i] == attr) { + put_mapping_n(mapping, 2); + return; + } + } + + gasket_nodev_error( + "Unable to put unknown attribute: %s", attr->attr.attr.name); +} +EXPORT_SYMBOL(gasket_sysfs_put_attr); + +ssize_t gasket_sysfs_register_show( + struct device *device, struct device_attribute *attr, char *buf) +{ + ulong reg_address, reg_bar, reg_value; + struct gasket_sysfs_mapping *mapping; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + + mapping = get_mapping(device); + if (!mapping) { + gasket_nodev_info("Device driver may have been removed."); + return 0; + } + + gasket_dev = mapping->gasket_dev; + if (!gasket_dev) { + gasket_nodev_error( + "No sysfs mapping found for device 0x%p", device); + put_mapping(mapping); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + put_mapping(mapping); + return 0; + } + + reg_address = gasket_attr->data.bar_address.offset; + reg_bar = gasket_attr->data.bar_address.bar; + reg_value = gasket_dev_read_64(gasket_dev, reg_bar, reg_address); + + gasket_sysfs_put_attr(device, gasket_attr); + put_mapping(mapping); + return snprintf(buf, PAGE_SIZE, "0x%lX\n", reg_value); +} +EXPORT_SYMBOL(gasket_sysfs_register_show); + +ssize_t gasket_sysfs_register_store( + struct device *device, struct device_attribute *attr, const char *buf, + size_t count) +{ + ulong parsed_value = 0; + struct gasket_sysfs_mapping *mapping; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + + if (count < 3 || buf[0] != '0' || buf[1] != 'x') { + gasket_nodev_error( + "sysfs register write format: \"0x\"."); + return -EINVAL; + } + + if (kstrtoul(buf, 16, &parsed_value) != 0) { + gasket_nodev_error( + "Unable to parse input as 64-bit hex value: %s.", buf); + return -EINVAL; + } + + mapping = get_mapping(device); + if (!mapping) { + gasket_nodev_info("Device driver may have been removed."); + return 0; + } + + gasket_dev = mapping->gasket_dev; + if (!gasket_dev) { + gasket_nodev_info("Device driver may have been removed."); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + put_mapping(mapping); + return count; + } + + gasket_dev_write_64(gasket_dev, parsed_value, + gasket_attr->data.bar_address.bar, + gasket_attr->data.bar_address.offset); + + if (gasket_attr->write_callback) + gasket_attr->write_callback( + gasket_dev, gasket_attr, parsed_value); + + gasket_sysfs_put_attr(device, gasket_attr); + put_mapping(mapping); + return count; +} +EXPORT_SYMBOL(gasket_sysfs_register_store); diff --git a/gasket_sysfs.h b/gasket_sysfs.h new file mode 100644 index 0000000..df9360e --- /dev/null +++ b/gasket_sysfs.h @@ -0,0 +1,199 @@ +/* Set of common sysfs utilities. + * + * Copyright (C) 2018 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* The functions described here are a set of utilities to allow each file in the + * Gasket driver framework to manage their own set of sysfs entries, instead of + * centralizing all that work in one file. + * + * The goal of these utilities is to allow for sysfs entries to be easily + * created without causing a proliferation of sysfs "show" functions. This + * requires O(N) string lookups during show function execution, but as reading + * sysfs entries is rarely performance-critical, this is likely acceptible. + */ +#ifndef __GASKET_SYSFS_H__ +#define __GASKET_SYSFS_H__ + +#include "gasket_constants.h" +#include "gasket_core.h" +#include +#include +#include + +/* The maximum number of mappings/devices a driver needs to support. */ +#define GASKET_SYSFS_NUM_MAPPINGS (GASKET_FRAMEWORK_DESC_MAX * GASKET_DEV_MAX) + +/* The maximum number of sysfs nodes in a directory. + */ +#define GASKET_SYSFS_MAX_NODES 196 + +/* End markers for sysfs struct arrays. */ +#define GASKET_ARRAY_END_TOKEN GASKET_RESERVED_ARRAY_END +#define GASKET_ARRAY_END_MARKER __stringify(GASKET_ARRAY_END_TOKEN) + +/* + * Terminator struct for a gasket_sysfs_attr array. Must be at the end of + * all gasket_sysfs_attribute arrays. + */ +#define GASKET_END_OF_ATTR_ARRAY \ + { \ + .attr = __ATTR(GASKET_ARRAY_END_TOKEN, S_IRUGO, NULL, NULL), \ + .data.attr_type = 0, \ + } + +/* + * Pairing of sysfs attribute and user data. + * Used in lookups in sysfs "show" functions to return attribute metadata. + */ +struct gasket_sysfs_attribute { + /* The underlying sysfs device attribute associated with this data. */ + struct device_attribute attr; + + /* User-specified data to associate with the attribute. */ + union { + struct bar_address_ { + ulong bar; + ulong offset; + } bar_address; + uint attr_type; + } data; + + /* + * Function pointer to a callback to be invoked when this attribute is + * written (if so configured). The arguments are to the Gasket device + * pointer, the enclosing gasket_attr structure, and the value written. + * The callback should perform any logging necessary, as errors cannot + * be returned from the callback. + */ + void (*write_callback)( + struct gasket_dev *dev, struct gasket_sysfs_attribute *attr, + ulong value); +}; + +#define GASKET_SYSFS_RO(_name, _show_function, _attr_type) \ + { \ + .attr = __ATTR(_name, S_IRUGO, _show_function, NULL), \ + .data.attr_type = _attr_type \ + } +#define GASKET_SYSFS_REG(_name, _offset, _bar) \ + { \ + .attr = __ATTR(_name, S_IRUGO, gasket_sysfs_register_show, \ + NULL), \ + .data.bar_address = { \ + .bar = _bar, \ + .offset = _offset \ + } \ + } + +/* Initializes the Gasket sysfs subsystem. + * + * Description: Performs one-time initialization. Must be called before usage + * at [Gasket] module load time. + */ +void gasket_sysfs_init(void); + +/* + * Create an entry in mapping_data between a device and a Gasket device. + * @device: Device struct to map to. + * @gasket_dev: The dev struct associated with the driver controlling @device. + * + * Description: This function maps a gasket_dev* to a device*. This mapping can + * be used in sysfs_show functions to get a handle to the gasket_dev struct + * controlling the device node. + * + * If this function is not called before gasket_sysfs_create_entries, a warning + * will be logged. + */ +int gasket_sysfs_create_mapping( + struct device *device, struct gasket_dev *gasket_dev); + +/* + * Creates bulk entries in sysfs. + * @device: Kernel device structure. + * @attrs: List of attributes/sysfs entries to create. + * + * Description: Creates each sysfs entry described in "attrs". Can be called + * multiple times for a given @device. If the gasket_dev specified in + * gasket_sysfs_create_mapping had a legacy device, the entries will be created + * for it, as well. + */ +int gasket_sysfs_create_entries( + struct device *device, const struct gasket_sysfs_attribute *attrs); + +/* + * Removes a device mapping from the global table. + * @device: Device to unmap. + * + * Description: Removes the device->Gasket device mapping from the internal + * table. + */ +void gasket_sysfs_remove_mapping(struct device *device); + +/* + * User data lookup based on kernel device structure. + * @device: Kernel device structure. + * + * Description: Returns the user data associated with "device" in a prior call + * to gasket_sysfs_create_entries. Returns NULL if no mapping can be found. + * Upon success, this call take a reference to internal sysfs data that must be + * released with gasket_sysfs_put_device_data. While this reference is held, the + * underlying device sysfs information/structure will remain valid/will not be + * deleted. + */ +struct gasket_dev *gasket_sysfs_get_device_data(struct device *device); + +/* + * Releases a references to internal data. + * @device: Kernel device structure. + * @dev: Gasket device descriptor (returned by gasket_sysfs_get_device_data). + */ +void gasket_sysfs_put_device_data( + struct device *device, struct gasket_dev *gasket_dev); + +/* + * Gasket-specific attribute lookup. + * @device: Kernel device structure. + * @attr: Device attribute to look up. + * + * Returns the Gasket sysfs attribute associated with the kernel device + * attribute and device structure itself. Upon success, this call will take a + * reference to internal sysfs data that must be released with a call to + * gasket_sysfs_get_device_data. While this reference is held, the underlying + * device sysfs information/structure will remain valid/will not be deleted. + */ +struct gasket_sysfs_attribute *gasket_sysfs_get_attr( + struct device *device, struct device_attribute *attr); + +/* + * Releases a references to internal data. + * @device: Kernel device structure. + * @attr: Gasket sysfs attribute descriptor (returned by + * gasket_sysfs_get_attr). + */ +void gasket_sysfs_put_attr( + struct device *device, struct gasket_sysfs_attribute *attr); + +/* Display a register as a sysfs node. */ +ssize_t gasket_sysfs_register_show( + struct device *device, struct device_attribute *attr, char *buf); + +/* + * Write to a register sysfs node. + * @buf: NULL-terminated data being written. + * @count: number of bytes in the "buf" argument. + */ +ssize_t gasket_sysfs_register_store( + struct device *device, struct device_attribute *attr, const char *buf, + size_t count); + +#endif /* __GASKET_SYSFS_H__ */ From e1c8e7a804c19b903da322189a75fbcd4d8d2635 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 4 Jul 2018 10:31:25 -0700 Subject: [PATCH 002/209] drivers/staging/gasket: Use 2-factor allocator calls As already done treewide, switch from open-coded multiplication to using 2-factor allocator helpers. Signed-off-by: Kees Cook Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 6 +++--- gasket_interrupt.c | 15 +++++++++------ gasket_page_table.c | 6 +++--- gasket_sysfs.c | 12 ++++++------ 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 6511a33..82b3eca 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1697,9 +1697,9 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) return -EPERM; } num_map_regions = bar_desc->num_mappable_regions; - map_regions = kzalloc( - num_map_regions * sizeof(*bar_desc->mappable_regions), - GFP_KERNEL); + map_regions = kcalloc(num_map_regions, + sizeof(*bar_desc->mappable_regions), + GFP_KERNEL); if (map_regions) { memcpy(map_regions, bar_desc->mappable_regions, num_map_regions * diff --git a/gasket_interrupt.c b/gasket_interrupt.c index b74eefe..1fd7bee 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -136,23 +136,26 @@ int gasket_interrupt_init( interrupt_data->wire_interrupt_offsets = wire_int_offsets; /* Allocate all dynamic structures. */ - interrupt_data->msix_entries = kzalloc( - sizeof(struct msix_entry) * num_interrupts, GFP_KERNEL); + interrupt_data->msix_entries = kcalloc(num_interrupts, + sizeof(struct msix_entry), + GFP_KERNEL); if (!interrupt_data->msix_entries) { kfree(interrupt_data); return -ENOMEM; } - interrupt_data->eventfd_ctxs = kzalloc( - sizeof(struct eventfd_ctx *) * num_interrupts, GFP_KERNEL); + interrupt_data->eventfd_ctxs = kcalloc(num_interrupts, + sizeof(struct eventfd_ctx *), + GFP_KERNEL); if (!interrupt_data->eventfd_ctxs) { kfree(interrupt_data->msix_entries); kfree(interrupt_data); return -ENOMEM; } - interrupt_data->interrupt_counts = kzalloc( - sizeof(ulong) * num_interrupts, GFP_KERNEL); + interrupt_data->interrupt_counts = kcalloc(num_interrupts, + sizeof(ulong), + GFP_KERNEL); if (!interrupt_data->interrupt_counts) { kfree(interrupt_data->eventfd_ctxs); kfree(interrupt_data->msix_entries); diff --git a/gasket_page_table.c b/gasket_page_table.c index 6dc1050..c5390a8 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1674,9 +1674,9 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, gasket_dev->page_table[index]->num_coherent_pages = num_pages; /* allocate the physical memory block */ - gasket_dev->page_table[index]->coherent_pages = kzalloc( - num_pages * sizeof(struct gasket_coherent_page_entry), - GFP_KERNEL); + gasket_dev->page_table[index]->coherent_pages = + kcalloc(num_pages, sizeof(struct gasket_coherent_page_entry), + GFP_KERNEL); if (!gasket_dev->page_table[index]->coherent_pages) goto nomem; *dma_address = 0; diff --git a/gasket_sysfs.c b/gasket_sysfs.c index d45098c..40268fb 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -137,9 +137,9 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) device = mapping->device; legacy_device = mapping->legacy_device; num_files_to_remove = mapping->attribute_count; - files_to_remove = kzalloc( - num_files_to_remove * sizeof(*files_to_remove), - GFP_KERNEL); + files_to_remove = kcalloc(num_files_to_remove, + sizeof(*files_to_remove), + GFP_KERNEL); for (i = 0; i < num_files_to_remove; i++) files_to_remove[i] = mapping->attributes[i].attr; @@ -238,9 +238,9 @@ int gasket_sysfs_create_mapping( kref_init(&mapping->refcount); mapping->device = device; mapping->gasket_dev = gasket_dev; - mapping->attributes = kzalloc( - GASKET_SYSFS_MAX_NODES * sizeof(*mapping->attributes), - GFP_KERNEL); + mapping->attributes = kcalloc(GASKET_SYSFS_MAX_NODES, + sizeof(*mapping->attributes), + GFP_KERNEL); mapping->attribute_count = 0; if (!mapping->attributes) { gasket_nodev_error("Unable to allocate sysfs attribute array."); From c8bfb85d08e3e6514ad247969cdc64abf1a42e20 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 6 Jul 2018 11:37:56 +0300 Subject: [PATCH 003/209] Staging: Gasket: uninitialized return in gasket_mmap() We forgot to set the error code on this error path so ret can be uninitialized. Fixes: 9a69f5087ccc ("drivers/staging: Gasket driver framework + Apex driver") Signed-off-by: Dan Carpenter Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 82b3eca..ad9442a 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1719,8 +1719,10 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) /* Try the next region if this one was not mappable. */ if (map_status == DO_MAP_REGION_INVALID) continue; - if (map_status == DO_MAP_REGION_FAILURE) + if (map_status == DO_MAP_REGION_FAILURE) { + ret = -ENOMEM; goto fail; + } has_mapped_anything = 1; } From 85a2ab89f40b907ec6e93f8bb50e0a04c4d78b50 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 6 Jul 2018 11:38:25 +0300 Subject: [PATCH 004/209] Staging: Gasket: fix a couple off by one bugs The > should be >= or we end up writing one element beyond the end of the interrupt_data->eventfd_ctxs[] array. Fixes: 9a69f5087ccc ("drivers/staging: Gasket driver framework + Apex driver") Signed-off-by: Dan Carpenter Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 1fd7bee..d1461b3 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -514,7 +514,7 @@ int gasket_interrupt_set_eventfd( if (IS_ERR(ctx)) return PTR_ERR(ctx); - if (interrupt < 0 || interrupt > interrupt_data->num_interrupts) + if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; interrupt_data->eventfd_ctxs[interrupt] = ctx; @@ -524,7 +524,7 @@ int gasket_interrupt_set_eventfd( int gasket_interrupt_clear_eventfd( struct gasket_interrupt_data *interrupt_data, int interrupt) { - if (interrupt < 0 || interrupt > interrupt_data->num_interrupts) + if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; interrupt_data->eventfd_ctxs[interrupt] = NULL; From 14da0230d8ad4f9c599703580b5c52de9b68851a Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 6 Jul 2018 11:39:21 +0300 Subject: [PATCH 005/209] Staging: Gasket: shift wrapping bug in gasket_read_modify_write_64() This function only has one caller so mask_width is 1 and mask_shift is 32. Shifting an int by 32 bits is undefined, but I guess on GCC it wraps to 0x1? Anyway it's supposed to be 0x100000000. Fixes: 9a69f5087ccc ("drivers/staging: Gasket driver framework + Apex driver") Signed-off-by: Dan Carpenter Signed-off-by: Greg Kroah-Hartman --- gasket_core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.h b/gasket_core.h index 203b9a3..5d6535a 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -683,7 +683,7 @@ static inline void gasket_read_modify_write_64( u64 mask, tmp; tmp = gasket_dev_read_64(dev, bar, location); - mask = ((1 << mask_width) - 1) << mask_shift; + mask = ((1ULL << mask_width) - 1) << mask_shift; tmp = (tmp & ~mask) | (value << mask_shift); gasket_dev_write_64(dev, tmp, bar, location); } From 392480bf3fcae170df69ba55d2aaf54c139c81d7 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Fri, 6 Jul 2018 23:43:10 -0700 Subject: [PATCH 006/209] drivers/staging/gasket: Use refcount_read() Use the refcount_read accessor function, avoid reaching into refcount and atomic struct fields. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 40268fb..a3705d6 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -122,7 +122,7 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) } mutex_lock(&mapping->mutex); - if (mapping->refcount.refcount.refs.counter == 0) + if (refcount_read(&mapping->refcount.refcount) == 0) gasket_nodev_error("Refcount is already 0!"); if (kref_put(&mapping->refcount, release_entry)) { gasket_nodev_info("Removing Gasket sysfs mapping, device %s", From dffd7b43598e7168b2073aa3af728ce07518f108 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 11 Jul 2018 11:40:05 +0100 Subject: [PATCH 007/209] staging: gasket: remove redundant pointer bar_data Pointer bar_data is being assigned but is never used hence it is redundant and can be removed. Cleans up clang warning: warning: variable 'bar_data' set but not used [-Wunused-but-set-variable] Signed-off-by: Colin Ian King Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index ad9442a..45914eb 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1610,7 +1610,6 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) bool is_coherent_region; const struct gasket_driver_desc *driver_desc; struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; - struct gasket_bar_data *bar_data; const struct gasket_bar_desc *bar_desc; struct gasket_mappable_region *map_regions = NULL; int num_map_regions = 0; @@ -1673,8 +1672,6 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) * Subtract the base of the bar from the raw offset to get the * memory location within the bar to map. */ - bar_data = &gasket_dev->bar_data[bar_index]; - bar_desc = &driver_desc->bar_descriptions[bar_index]; permissions = bar_desc->permissions; if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) { From cc8a3327dac772c39ba2989f50f7b23da0b02deb Mon Sep 17 00:00:00 2001 From: Ivan Bornyakov Date: Wed, 11 Jul 2018 14:13:34 +0300 Subject: [PATCH 008/209] staging: gasket: fix plain integer as NULL pointer warning Trivial fix to remove sparse warnings: drivers/staging/gasket/gasket_page_table.c:884:40: warning: Using plain integer as NULL pointer drivers/staging/gasket/gasket_page_table.c:1743:57: warning: Using plain integer as NULL pointer drivers/staging/gasket/gasket_page_table.c:1768:57: warning: Using plain integer as NULL pointer Signed-off-by: Ivan Bornyakov Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index c5390a8..5d3d33c 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -881,7 +881,7 @@ static int gasket_perform_mapping( u64 off = (u64)host_addr - (u64)pg_tbl->coherent_pages[0].user_virt; - ptes[i].page = 0; + ptes[i].page = NULL; ptes[i].offset = offset; ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + off + i * PAGE_SIZE; @@ -1740,7 +1740,7 @@ int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, gasket_dev->coherent_buffer.virt_base, gasket_dev->coherent_buffer.phys_base); gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = 0; + gasket_dev->coherent_buffer.virt_base = NULL; gasket_dev->coherent_buffer.phys_base = 0; } return 0; @@ -1765,7 +1765,7 @@ void gasket_free_coherent_memory_all( gasket_dev->coherent_buffer.virt_base, gasket_dev->coherent_buffer.phys_base); gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = 0; + gasket_dev->coherent_buffer.virt_base = NULL; gasket_dev->coherent_buffer.phys_base = 0; } } From 8ff3da62cbe580a835cfb6a97fe9404e5e3b704a Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Wed, 11 Jul 2018 13:39:10 +0200 Subject: [PATCH 009/209] staging: gasket: add SPDX identifiers to all files. It's good to have SPDX identifiers in all files to make it easier to audit the kernel tree for correct licenses. Fix up the all of the staging gasket files to have a proper SPDX identifier, based on the license text in the file itself. The SPDX identifier is a legally binding shorthand, which can be used instead of the full boiler plate text. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- apex.h | 1 + apex_driver.c | 1 + gasket.h | 1 + gasket_constants.h | 1 + gasket_core.c | 1 + gasket_core.h | 1 + gasket_interrupt.c | 1 + gasket_interrupt.h | 1 + gasket_ioctl.c | 1 + gasket_ioctl.h | 1 + gasket_logging.h | 1 + gasket_page_table.c | 1 + gasket_page_table.h | 1 + gasket_sysfs.c | 1 + gasket_sysfs.h | 1 + 15 files changed, 15 insertions(+) diff --git a/apex.h b/apex.h index f2600aa..1d1f34d 100644 --- a/apex.h +++ b/apex.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Apex kernel-userspace interface definition(s). * diff --git a/apex_driver.c b/apex_driver.c index 3952567..670ada3 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Driver for the Apex chip. * * Copyright (C) 2018 Google, Inc. diff --git a/gasket.h b/gasket.h index 593d508..c0ea9ad 100644 --- a/gasket.h +++ b/gasket.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Common Gasket device kernel and user space declarations. * * Copyright (C) 2018 Google, Inc. diff --git a/gasket_constants.h b/gasket_constants.h index b39e3e3..e70c222 100644 --- a/gasket_constants.h +++ b/gasket_constants.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2018 Google, Inc. * * This software is licensed under the terms of the GNU General Public diff --git a/gasket_core.c b/gasket_core.c index 45914eb..a09f491 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Gasket generic driver framework. This file contains the implementation * for the Gasket generic driver framework - the functionality that is common * across Gasket devices. diff --git a/gasket_core.h b/gasket_core.h index 5d6535a..be25d93 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Gasket generic driver. Defines the set of data types and functions necessary * to define a driver using the Gasket generic driver framework. * diff --git a/gasket_interrupt.c b/gasket_interrupt.c index d1461b3..28bb80d 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2018 Google, Inc. * * This software is licensed under the terms of the GNU General Public diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 3a8afae..2220ca4 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Gasket common interrupt module. Defines functions for enabling * eventfd-triggered interrupts between a Gasket device and a host process. diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 4758083..c5d7bee 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2018 Google, Inc. * * This software is licensed under the terms of the GNU General Public diff --git a/gasket_ioctl.h b/gasket_ioctl.h index df86800..457b316 100644 --- a/gasket_ioctl.h +++ b/gasket_ioctl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2018 Google, Inc. * * This software is licensed under the terms of the GNU General Public diff --git a/gasket_logging.h b/gasket_logging.h index fa17b4a..e288da6 100644 --- a/gasket_logging.h +++ b/gasket_logging.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Common logging utilities for the Gasket driver framework. * * Copyright (C) 2018 Google, Inc. diff --git a/gasket_page_table.c b/gasket_page_table.c index 5d3d33c..e86c160 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Implementation of Gasket page table support. * * Copyright (C) 2018 Google, Inc. diff --git a/gasket_page_table.h b/gasket_page_table.h index f2f519a..2074239 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Gasket Page Table functionality. This file describes the address * translation/paging functionality supported by the Gasket driver framework. * As much as possible, internal details are hidden to simplify use - diff --git a/gasket_sysfs.c b/gasket_sysfs.c index a3705d6..b16ecac 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2018 Google, Inc. * * This software is licensed under the terms of the GNU General Public diff --git a/gasket_sysfs.h b/gasket_sysfs.h index df9360e..26aa091 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Set of common sysfs utilities. * * Copyright (C) 2018 Google, Inc. From 653a5901f5bc8203dfe155c3c8124eef218f391e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Wed, 11 Jul 2018 13:39:11 +0200 Subject: [PATCH 010/209] staging: gasket: remove redundant license information Now that the SPDX tag is in all gasket files, that identifies the license in a specific and legally-defined manner. So the extra GPL text wording can be removed as it is no longer needed at all. This is done on a quest to remove the 700+ different ways that files in the kernel describe the GPL license text. And there's unneeded stuff like the address (sometimes incorrect) for the FSF which is never needed. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- apex.h | 9 --------- apex_driver.c | 12 ++---------- gasket.h | 12 ++---------- gasket_constants.h | 12 +----------- gasket_core.c | 12 ++---------- gasket_core.h | 12 ++---------- gasket_interrupt.c | 12 +----------- gasket_interrupt.h | 9 --------- gasket_ioctl.c | 12 +----------- gasket_ioctl.h | 12 +----------- gasket_logging.h | 12 ++---------- gasket_page_table.c | 12 ++---------- gasket_page_table.h | 12 ++---------- gasket_sysfs.c | 12 +----------- gasket_sysfs.h | 12 ++---------- 15 files changed, 21 insertions(+), 153 deletions(-) diff --git a/apex.h b/apex.h index 1d1f34d..4ef2641 100644 --- a/apex.h +++ b/apex.h @@ -3,15 +3,6 @@ * Apex kernel-userspace interface definition(s). * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __APEX_H__ #define __APEX_H__ diff --git a/apex_driver.c b/apex_driver.c index 670ada3..e2258e4 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -1,16 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 -/* Driver for the Apex chip. +/* + * Driver for the Apex chip. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/gasket.h b/gasket.h index c0ea9ad..9f709f0 100644 --- a/gasket.h +++ b/gasket.h @@ -1,16 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Common Gasket device kernel and user space declarations. +/* + * Common Gasket device kernel and user space declarations. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __GASKET_H__ #define __GASKET_H__ diff --git a/gasket_constants.h b/gasket_constants.h index e70c222..82ed3f2 100644 --- a/gasket_constants.h +++ b/gasket_constants.h @@ -1,15 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* Copyright (C) 2018 Google, Inc. */ #ifndef __GASKET_CONSTANTS_H__ #define __GASKET_CONSTANTS_H__ diff --git a/gasket_core.c b/gasket_core.c index a09f491..926113b 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1,18 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 -/* Gasket generic driver framework. This file contains the implementation +/* + * Gasket generic driver framework. This file contains the implementation * for the Gasket generic driver framework - the functionality that is common * across Gasket devices. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include "gasket_core.h" diff --git a/gasket_core.h b/gasket_core.h index be25d93..4501344 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -1,17 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Gasket generic driver. Defines the set of data types and functions necessary +/* + * Gasket generic driver. Defines the set of data types and functions necessary * to define a driver using the Gasket generic driver framework. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __GASKET_CORE_H__ #define __GASKET_CORE_H__ diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 28bb80d..faaabdc 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -1,15 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* Copyright (C) 2018 Google, Inc. */ #include "gasket_interrupt.h" diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 2220ca4..44ea985 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -4,15 +4,6 @@ * eventfd-triggered interrupts between a Gasket device and a host process. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __GASKET_INTERRUPT_H__ #define __GASKET_INTERRUPT_H__ diff --git a/gasket_ioctl.c b/gasket_ioctl.c index c5d7bee..3c54542 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -1,15 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* Copyright (C) 2018 Google, Inc. */ #include "gasket.h" #include "gasket_ioctl.h" #include "gasket_constants.h" diff --git a/gasket_ioctl.h b/gasket_ioctl.h index 457b316..461fab2 100644 --- a/gasket_ioctl.h +++ b/gasket_ioctl.h @@ -1,15 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* Copyright (C) 2018 Google, Inc. */ #ifndef __GASKET_IOCTL_H__ #define __GASKET_IOCTL_H__ diff --git a/gasket_logging.h b/gasket_logging.h index e288da6..54bbe51 100644 --- a/gasket_logging.h +++ b/gasket_logging.h @@ -1,16 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Common logging utilities for the Gasket driver framework. +/* + * Common logging utilities for the Gasket driver framework. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include diff --git a/gasket_page_table.c b/gasket_page_table.c index e86c160..f00a8f1 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1,16 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 -/* Implementation of Gasket page table support. +/* + * Implementation of Gasket page table support. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ /* diff --git a/gasket_page_table.h b/gasket_page_table.h index 2074239..d8d031c 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -1,20 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Gasket Page Table functionality. This file describes the address +/* + * Gasket Page Table functionality. This file describes the address * translation/paging functionality supported by the Gasket driver framework. * As much as possible, internal details are hidden to simplify use - * all calls are thread-safe (protected by an internal mutex) except where * indicated otherwise. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __GASKET_ADDR_TRNSL_H__ diff --git a/gasket_sysfs.c b/gasket_sysfs.c index b16ecac..94e55f1 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -1,15 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +/* Copyright (C) 2018 Google, Inc. */ #include "gasket_sysfs.h" #include "gasket_core.h" diff --git a/gasket_sysfs.h b/gasket_sysfs.h index 26aa091..40db5a0 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -1,16 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Set of common sysfs utilities. +/* + * Set of common sysfs utilities. * * Copyright (C) 2018 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ /* The functions described here are a set of utilities to allow each file in the From 57bc89e0ef8f345818ffc457d33e764e231a4711 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Wed, 11 Jul 2018 13:39:12 +0200 Subject: [PATCH 011/209] staging: gasket: remove TODO item about SPDX usage Now that the files are all properly tagged with SPDX lines, and the boilerplate license text is gone, remove the TODO item. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- TODO | 2 -- 1 file changed, 2 deletions(-) diff --git a/TODO b/TODO index 0d8ee96..d3c44ca 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,5 @@ This is a list of things that need to be done to get this driver out of the staging directory. -- Use SPDX tags to show the license of the file, and no more "boiler-plate" - license text is needed. - Remove static function declarations. - Document sysfs files with Documentation/ABI/ entries. - Use misc interface instead of major number for driver version description. From 4ec0581576f194d30ccd565bdead8381e421515e Mon Sep 17 00:00:00 2001 From: Felix Siegel Date: Thu, 12 Jul 2018 21:27:13 +0200 Subject: [PATCH 012/209] staging: gasket: Move open-curly brace to match kernel code style Move open open-curly brace to the next line following function definition to match the kernel's coding style Signed-off-by: Felix Siegel Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 926113b..5863a3f 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1895,7 +1895,8 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) EXPORT_SYMBOL(gasket_reset_nolock); gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( - struct gasket_dev *gasket_dev) { + struct gasket_dev *gasket_dev) +{ return gasket_dev->internal_desc->driver_desc->ioctl_permissions_cb; } EXPORT_SYMBOL(gasket_get_ioctl_permissions_cb); From c2b4e1cb8d74e8b0f800178e0cdfd88c490af850 Mon Sep 17 00:00:00 2001 From: Felix Siegel Date: Thu, 12 Jul 2018 21:27:14 +0200 Subject: [PATCH 013/209] staging: gasket: fix multi line comments style This patch fixes checkpatch.pl warnings: WARNING: Block comments should align the * on each line Signed-off-by: Felix Siegel Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 5863a3f..4ca6e53 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1347,9 +1347,9 @@ static bool gasket_mm_get_mapping_addrs( *virt_offset = 0; if (bar_offset + requested_length < range_start) { /* - * If the requested region is completely below the range, - * there is nothing to map. - */ + * If the requested region is completely below the range, + * there is nothing to map. + */ return false; } else if (bar_offset <= range_start) { /* If the bar offset is below this range's start @@ -1507,7 +1507,7 @@ fail: * Calculates the offset where the VMA range begins in its containing BAR. * The offset is written into bar_offset on success. * Returns zero on success, anything else on error. -*/ + */ static int gasket_mm_vma_bar_offset( const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, ulong *bar_offset) From dbd32599d20ae8426e8afeaa51921d1d2fc86f35 Mon Sep 17 00:00:00 2001 From: Felix Siegel Date: Fri, 13 Jul 2018 00:58:48 +0200 Subject: [PATCH 014/209] staging: gasket: remove "function entered" log messages Remove log messages that solely print the function's name everytime it is called. Signed-off-by: Felix Siegel Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index e2258e4..161c48b 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -341,8 +341,6 @@ static int apex_add_dev_cb(struct gasket_dev *gasket_dev) ulong page_table_ready, msix_table_ready; int retries = 0; - gasket_log_error(gasket_dev, "apex_add_dev_cb."); - apex_reset(gasket_dev, 0); while (retries < APEX_RESET_RETRY) { @@ -447,8 +445,6 @@ static int apex_reset(struct gasket_dev *gasket_dev, uint type) if (bypass_top_level) return 0; - gasket_log_debug(gasket_dev, "apex_reset."); - if (!is_gcb_in_reset(gasket_dev)) { /* We are not in reset - toggle the reset bit so as to force * re-init of custom block @@ -472,8 +468,6 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) if (bypass_top_level) return 0; - gasket_log_debug(gasket_dev, "apex_enter_reset."); - /* * Software reset: * Enable sleep mode @@ -534,8 +528,6 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) if (bypass_top_level) return 0; - gasket_log_debug(gasket_dev, "apex_quit_reset."); - /* * Disable sleep mode: * - Disable GCB memory shut down: From b27f5c2e98b261dccdbfda5bfa80eea111d23492 Mon Sep 17 00:00:00 2001 From: Felix Siegel Date: Fri, 13 Jul 2018 00:58:49 +0200 Subject: [PATCH 015/209] staging: gasket: Use __func__ instead of hardcoded string - Style Changed logging statements to use %s and __func__ instead of hard coding the function name in a string. Signed-off-by: Felix Siegel Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 7 ++++--- gasket_core.c | 8 +++++--- gasket_ioctl.c | 3 ++- gasket_page_table.c | 13 ++++++++----- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 161c48b..cca4cf4 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -420,8 +420,9 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) gasket_log_info( gasket_dev, - "apex_device_cleanup 0x%p hib_error 0x%llx scalar_error " + "%s 0x%p hib_error 0x%llx scalar_error " "0x%llx.", + __func__, gasket_dev, hib_error, scalar_error); if (allow_power_save) @@ -449,7 +450,7 @@ static int apex_reset(struct gasket_dev *gasket_dev, uint type) /* We are not in reset - toggle the reset bit so as to force * re-init of custom block */ - gasket_log_debug(gasket_dev, "apex_reset: toggle reset."); + gasket_log_debug(gasket_dev, "%s: toggle reset.", __func__); ret = apex_enter_reset(gasket_dev, type); if (ret) @@ -673,7 +674,7 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg) return -EFAULT; gasket_log_error( - gasket_dev, "apex_clock_gating %llu", ibuf.enable); + gasket_dev, "%s %llu", __func__, ibuf.enable); if (ibuf.enable) { /* Quiesce AXI, gate GCB clock. */ diff --git a/gasket_core.c b/gasket_core.c index 4ca6e53..14649a7 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -932,7 +932,7 @@ static int gasket_enable_dev( } else { gasket_log_error( gasket_dev, - "gasket_enable_dev with no physical device!!"); + "%s with no physical device!!", __func__); WARN_ON(1); ddev = NULL; } @@ -2100,8 +2100,9 @@ int gasket_wait_sync( if (diff_nanosec > timeout_ns) { gasket_log_error( gasket_dev, - "gasket_wait_sync timeout: reg %llx count %x " + "%s timeout: reg %llx count %x " "dma %lld ns\n", + __func__, offset, count, diff_nanosec); return -1; } @@ -2141,7 +2142,8 @@ int gasket_wait_with_reschedule( if (retries == max_retries) { gasket_log_error( gasket_dev, - "gasket_wait_with_reschedule timeout: reg %llx timeout (%llu ms)", + "%s timeout: reg %llx timeout (%llu ms)", + __func__, offset, max_retries * delay_ms); return -EINVAL; } diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 3c54542..0c2f85c 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -178,7 +178,8 @@ static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd) alive = (gasket_dev->status == GASKET_STATUS_ALIVE); if (!alive) { gasket_nodev_error( - "gasket_ioctl_check_permissions alive %d status %d.", + "%s alive %d status %d.", + __func__, alive, gasket_dev->status); } diff --git a/gasket_page_table.c b/gasket_page_table.c index f00a8f1..3de7f8c 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -445,8 +445,9 @@ int gasket_page_table_map( mutex_unlock(&pg_tbl->mutex); gasket_nodev_debug( - "gasket_page_table_map done: ha %llx daddr %llx num %d, " + "%s done: ha %llx daddr %llx num %d, " "ret %d\n", + __func__, (unsigned long long)host_addr, (unsigned long long)dev_addr, num_pages, ret); return ret; @@ -869,7 +870,7 @@ static int gasket_perform_mapping( for (i = 0; i < num_pages; i++) { page_addr = host_addr + i * PAGE_SIZE; offset = page_addr & (PAGE_SIZE - 1); - gasket_nodev_debug("gasket_perform_mapping i %d\n", i); + gasket_nodev_debug("%s i %d\n", __func__, i); if (is_coherent(pg_tbl, host_addr)) { u64 off = (u64)host_addr - @@ -907,17 +908,19 @@ static int gasket_perform_mapping( } gasket_nodev_debug( - " gasket_perform_mapping dev %p " + "%s dev %p " "i %d pte %p pfn %p -> mapped %llx\n", + __func__, pg_tbl->device, i, &ptes[i], (void *)page_to_pfn(page), (unsigned long long)ptes[i].dma_addr); if (ptes[i].dma_addr == -1) { gasket_nodev_error( - "gasket_perform_mapping i %d" + "%s i %d" " -> fail to map page %llx " "[pfn %p ohys %p]\n", + __func__, i, (unsigned long long)ptes[i].dma_addr, (void *)page_to_pfn(page), @@ -1623,7 +1626,7 @@ int gasket_set_user_virt( pg_tbl = gasket_dev->page_table[0]; if (!pg_tbl) { gasket_nodev_error( - "gasket_set_user_virt: invalid page table index"); + "%s: invalid page table index", __func__); return 0; } for (j = 0; j < num_pages; j++) { From 533507835720538d2376972e770afd8f164f9f5c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 12:05:51 +0200 Subject: [PATCH 016/209] staging: gasket: remove pointless gasket_interrupt_pause() gasket_interrupt_pause() does nothing, and no one calls it, so remove it as it is dead-weight. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 18 ------------------ gasket_interrupt.h | 11 ----------- 2 files changed, 29 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index faaabdc..922ffd5 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -426,24 +426,6 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } /* See gasket_interrupt.h for description. */ -void gasket_interrupt_pause(struct gasket_dev *gasket_dev, int enable_pause) -{ - WARN_ON(!gasket_dev); - - if (!gasket_dev->interrupt_data) - return; /* nothing to do */ - - if (gasket_dev->interrupt_data->type == PCI_MSI || - gasket_dev->interrupt_data->type == PCI_MSIX) { - /* Nothing to be done for MSI/MSIX just yet. */ - } - - if (gasket_dev->interrupt_data->type == PLATFORM_WIRE) { - /* Nothing to be done for PLATFORM_WIRE */ - } -} -EXPORT_SYMBOL(gasket_interrupt_pause); - void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) { struct gasket_interrupt_data *interrupt_data = diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 44ea985..44ceede 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -150,15 +150,4 @@ struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs( int gasket_interrupt_system_status(struct gasket_dev *gasket_dev); -/* - * Masks interrupts and de-register the handler. - * After an interrupt pause it is not guaranteed that the chip registers will - * be accessible anymore, since the chip may be in a power save mode, - * which means that the interrupt handler (if it were to happen) may not - * have a way to clear the interrupt condition. - * @gasket_dev: The Gasket device struct - * @enable_pause: Whether to pause or unpause the interrupts. - */ -void gasket_interrupt_pause(struct gasket_dev *gasket_dev, int enable_pause); - #endif From e7d963fa612f6c6b5336298acf7bed16d574c73c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 12:05:52 +0200 Subject: [PATCH 017/209] staging: gasket: remove gasket_interrupt_get_eventfd_ctxs() It is exported, yet no one calls it so just remove the dead code. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 7 ------- gasket_interrupt.h | 12 ------------ 2 files changed, 19 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 922ffd5..3ea168f 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -522,13 +522,6 @@ struct msix_entry *gasket_interrupt_get_msix_entries( return interrupt_data->msix_entries; } -struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs( - struct gasket_interrupt_data *interrupt_data) -{ - return interrupt_data->eventfd_ctxs; -} -EXPORT_SYMBOL(gasket_interrupt_get_eventfd_ctxs); - static ssize_t interrupt_sysfs_show( struct device *device, struct device_attribute *attr, char *buf) { diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 44ceede..f0586a6 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -128,18 +128,6 @@ int gasket_interrupt_trigger_eventfd( struct msix_entry *gasket_interrupt_get_msix_entries( struct gasket_interrupt_data *interrupt_data); -/* - * Get a pointer to data's eventfd contexts. - * @data: The interrupt data from which to extract. - * - * Returns the internal pointer to data's eventfd contexts. - * - * This function exists for backwards compatibility with older drivers. - * No new uses should be written. - */ -struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs( - struct gasket_interrupt_data *interrupt_data); - /* * Get the health of the interrupt subsystem. * @gasket_dev: The Gasket device struct. From 97ca04034274fba54f023f53d549c947d50618e0 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 12:05:53 +0200 Subject: [PATCH 018/209] staging: gasket: remove gasket_interrupt_trigger_eventfd() No one calls it, so just remove the dead code. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 12 ------------ gasket_interrupt.h | 13 ------------- 2 files changed, 25 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 3ea168f..691996d 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -504,18 +504,6 @@ int gasket_interrupt_clear_eventfd( return 0; } -int gasket_interrupt_trigger_eventfd( - struct gasket_interrupt_data *interrupt_data, int interrupt) -{ - struct eventfd_ctx *ctx = interrupt_data->eventfd_ctxs[interrupt]; - - if (!ctx) - return -EINVAL; - - eventfd_signal(ctx, 1); - return 0; -} - struct msix_entry *gasket_interrupt_get_msix_entries( struct gasket_interrupt_data *interrupt_data) { diff --git a/gasket_interrupt.h b/gasket_interrupt.h index f0586a6..5b65d05 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -101,19 +101,6 @@ int gasket_interrupt_set_eventfd( int gasket_interrupt_clear_eventfd( struct gasket_interrupt_data *interrupt_data, int interrupt); -/* - * Signals the eventfd associated with interrupt. - * @data: Pointer to device interrupt data. - * @interrupt: The device interrupt to signal for. - * - * Simulates a device interrupt by signaling the eventfd associated with - * interrupt, if any. - * Returns 0 if the eventfd was successfully triggered, a negative error code - * otherwise (if, for example, no eventfd was associated with interrupt). - */ -int gasket_interrupt_trigger_eventfd( - struct gasket_interrupt_data *interrupt_data, int interrupt); - /* * The below functions exist for backwards compatibility. * No new uses should be written. From 2343ad2d6197fa86077f47f4ff264fd67ded509c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 12:05:54 +0200 Subject: [PATCH 019/209] staging: gasket: remove gasket_interrupt_get_msix_entries() No one calls it, it is claimed to be "legacy", whatever that means, so just remove the dead code. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 6 ------ gasket_interrupt.h | 10 ---------- 2 files changed, 16 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 691996d..d096ce7 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -504,12 +504,6 @@ int gasket_interrupt_clear_eventfd( return 0; } -struct msix_entry *gasket_interrupt_get_msix_entries( - struct gasket_interrupt_data *interrupt_data) -{ - return interrupt_data->msix_entries; -} - static ssize_t interrupt_sysfs_show( struct device *device, struct device_attribute *attr, char *buf) { diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 5b65d05..805fee6 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -105,16 +105,6 @@ int gasket_interrupt_clear_eventfd( * The below functions exist for backwards compatibility. * No new uses should be written. */ -/* - * Retrieve a pointer to data's MSI-X - * entries. - * @data: The interrupt data from which to extract. - * - * Returns the internal pointer to data's MSI-X entries. - */ -struct msix_entry *gasket_interrupt_get_msix_entries( - struct gasket_interrupt_data *interrupt_data); - /* * Get the health of the interrupt subsystem. * @gasket_dev: The Gasket device struct. From 8ab2804fadfbb2430009d32b40cf18d11564ce60 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 12:05:55 +0200 Subject: [PATCH 020/209] staging: gasket: remove gasket_page_table_num_extended_entries() It is exported, yet no one calls it so just remove the dead code. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 11 ----------- gasket_page_table.h | 7 ------- 2 files changed, 18 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 3de7f8c..dcd52e1 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -640,17 +640,6 @@ uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl) EXPORT_SYMBOL(gasket_page_table_num_simple_entries); /* See gasket_page_table.h for description. */ -uint gasket_page_table_num_extended_entries(struct gasket_page_table *pg_tbl) -{ - if (!pg_tbl) { - gasket_nodev_error("Passed a null page table."); - return 0; - } - - return pg_tbl->num_extended_entries; -} -EXPORT_SYMBOL(gasket_page_table_num_extended_entries); - uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl) { if (!pg_tbl) { diff --git a/gasket_page_table.h b/gasket_page_table.h index d8d031c..720ce2b 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -201,13 +201,6 @@ uint gasket_page_table_num_entries(struct gasket_page_table *page_table); */ uint gasket_page_table_num_simple_entries(struct gasket_page_table *page_table); -/* - * Gets the number of extended entries. - * @page_table: Gasket page table pointer. - */ -uint gasket_page_table_num_extended_entries( - struct gasket_page_table *page_table); - /* * Gets the number of actively pinned pages. * @page_table: Gasket page table pointer. From e5fd02987e9a0ab2597383fe56ebc15ddfa0f22b Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 10:29:02 +0200 Subject: [PATCH 021/209] staging: gasket: remove GASKET_SYSFS_REG() In an attempt to start to clean up the monstrosity of the sysfs abuse in the gasket driver, let's remove code that is not used at all. The GASKET_SYSFS_REG() macro is never used, so delete it. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gasket_sysfs.h b/gasket_sysfs.h index 40db5a0..0949cda 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -78,15 +78,6 @@ struct gasket_sysfs_attribute { .attr = __ATTR(_name, S_IRUGO, _show_function, NULL), \ .data.attr_type = _attr_type \ } -#define GASKET_SYSFS_REG(_name, _offset, _bar) \ - { \ - .attr = __ATTR(_name, S_IRUGO, gasket_sysfs_register_show, \ - NULL), \ - .data.bar_address = { \ - .bar = _bar, \ - .offset = _offset \ - } \ - } /* Initializes the Gasket sysfs subsystem. * From 1180baae6b8666723fb839a66c2fb070d1daea3c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 10:29:03 +0200 Subject: [PATCH 022/209] staging: gasket: remove gasket_sysfs_register_show() In an attempt to start to clean up the monstrosity of the sysfs abuse in the gasket driver, let's remove code that is not used at all. The gasket_sysfs_register_show() function is never used, so delete it. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 38 -------------------------------------- gasket_sysfs.h | 4 ---- 2 files changed, 42 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 94e55f1..39f9595 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -396,44 +396,6 @@ void gasket_sysfs_put_attr( } EXPORT_SYMBOL(gasket_sysfs_put_attr); -ssize_t gasket_sysfs_register_show( - struct device *device, struct device_attribute *attr, char *buf) -{ - ulong reg_address, reg_bar, reg_value; - struct gasket_sysfs_mapping *mapping; - struct gasket_dev *gasket_dev; - struct gasket_sysfs_attribute *gasket_attr; - - mapping = get_mapping(device); - if (!mapping) { - gasket_nodev_info("Device driver may have been removed."); - return 0; - } - - gasket_dev = mapping->gasket_dev; - if (!gasket_dev) { - gasket_nodev_error( - "No sysfs mapping found for device 0x%p", device); - put_mapping(mapping); - return 0; - } - - gasket_attr = gasket_sysfs_get_attr(device, attr); - if (!gasket_attr) { - put_mapping(mapping); - return 0; - } - - reg_address = gasket_attr->data.bar_address.offset; - reg_bar = gasket_attr->data.bar_address.bar; - reg_value = gasket_dev_read_64(gasket_dev, reg_bar, reg_address); - - gasket_sysfs_put_attr(device, gasket_attr); - put_mapping(mapping); - return snprintf(buf, PAGE_SIZE, "0x%lX\n", reg_value); -} -EXPORT_SYMBOL(gasket_sysfs_register_show); - ssize_t gasket_sysfs_register_store( struct device *device, struct device_attribute *attr, const char *buf, size_t count) diff --git a/gasket_sysfs.h b/gasket_sysfs.h index 0949cda..e9f4fad 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -167,10 +167,6 @@ struct gasket_sysfs_attribute *gasket_sysfs_get_attr( void gasket_sysfs_put_attr( struct device *device, struct gasket_sysfs_attribute *attr); -/* Display a register as a sysfs node. */ -ssize_t gasket_sysfs_register_show( - struct device *device, struct device_attribute *attr, char *buf); - /* * Write to a register sysfs node. * @buf: NULL-terminated data being written. From 1ba805dba809d86ed320070ee423e7e2a14a2f8e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 10:29:04 +0200 Subject: [PATCH 023/209] staging: gasket: sysfs: remove legacy_device field This field is only ever checked, never actually set, and looks to be left-over from some old interface of some sort. As it's not being used at all here, and is just adding to the complexity, delete it. Cc: Rob Springer Cc: John Joseph Cc: Ben Chan Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 39f9595..e3d7706 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -17,9 +17,6 @@ struct gasket_sysfs_mapping { */ struct device *device; - /* Legacy device struct, if used by this mapping's driver. */ - struct device *legacy_device; - /* The Gasket descriptor for this device. */ struct gasket_dev *gasket_dev; @@ -75,8 +72,7 @@ static struct gasket_sysfs_mapping *get_mapping(struct device *device) for (i = 0; i < GASKET_SYSFS_NUM_MAPPINGS; i++) { mutex_lock(&dev_mappings[i].mutex); - if (dev_mappings[i].device == device || - dev_mappings[i].legacy_device == device) { + if (dev_mappings[i].device == device) { kref_get(&dev_mappings[i].refcount); mutex_unlock(&dev_mappings[i].mutex); return &dev_mappings[i]; @@ -105,7 +101,6 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) int num_files_to_remove = 0; struct device_attribute *files_to_remove; struct device *device; - struct device *legacy_device; if (!mapping) { gasket_nodev_info("Mapping should not be NULL."); @@ -126,7 +121,6 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) * sysfs nodes are removed outside the lock. */ device = mapping->device; - legacy_device = mapping->legacy_device; num_files_to_remove = mapping->attribute_count; files_to_remove = kcalloc(num_files_to_remove, sizeof(*files_to_remove), @@ -143,12 +137,8 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) mutex_unlock(&mapping->mutex); if (num_files_to_remove != 0) { - for (i = 0; i < num_files_to_remove; ++i) { + for (i = 0; i < num_files_to_remove; ++i) device_remove_file(device, &files_to_remove[i]); - if (legacy_device) - device_remove_file( - legacy_device, &files_to_remove[i]); - } kfree(files_to_remove); } } @@ -282,21 +272,6 @@ int gasket_sysfs_create_entries( return ret; } - if (mapping->legacy_device) { - ret = device_create_file(mapping->legacy_device, - &attrs[i].attr); - if (ret) { - gasket_log_error( - mapping->gasket_dev, - "Unable to create legacy sysfs entries;" - " rc: %d", - ret); - mutex_unlock(&mapping->mutex); - put_mapping(mapping); - return ret; - } - } - mapping->attributes[mapping->attribute_count] = attrs[i]; ++mapping->attribute_count; } From 39bdbb8951ddb46266653b452805f4a473bf24e0 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 13 Jul 2018 17:46:17 +0200 Subject: [PATCH 024/209] staging: gasket: remove gasket_wait_sync() This function is not called anywhere, so just remove it. Also, as an added benifit, Arnd points out that it doesn't even work properly: This code won't work correct during leap seconds or a concurrent settimeofday() call, and it probably doesn't do what the author intended even for the normal case, as it passes a timeout in nanoseconds but reads the time using a jiffies-granularity accessor. Reported-by: Arnd Bergmann Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 45 --------------------------------------------- gasket_core.h | 5 ----- 2 files changed, 50 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 14649a7..b69b630 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -2067,51 +2067,6 @@ struct device *gasket_get_device(struct gasket_dev *dev) return NULL; } -/** - * Synchronously waits on device. - * @gasket_dev: Device struct. - * @bar: Bar - * @offset: Register offset - * @mask: Register mask - * @val: Expected value - * @timeout_ns: Timeout in nanoseconds - * - * Description: Busy waits for a specific combination of bits to be set - * on a Gasket register. - **/ -int gasket_wait_sync( - struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, - u64 timeout_ns) -{ - u64 reg; - struct timespec start_time, cur_time; - u64 diff_nanosec; - int count = 0; - - reg = gasket_dev_read_64(gasket_dev, bar, offset); - start_time = current_kernel_time(); - while ((reg & mask) != val) { - count++; - cur_time = current_kernel_time(); - diff_nanosec = (u64)(cur_time.tv_sec - start_time.tv_sec) * - 1000000000LL + - (u64)(cur_time.tv_nsec) - - (u64)(start_time.tv_nsec); - if (diff_nanosec > timeout_ns) { - gasket_log_error( - gasket_dev, - "%s timeout: reg %llx count %x " - "dma %lld ns\n", - __func__, - offset, count, diff_nanosec); - return -1; - } - reg = gasket_dev_read_64(gasket_dev, bar, offset); - } - return 0; -} -EXPORT_SYMBOL(gasket_wait_sync); - /** * Asynchronously waits on device. * @gasket_dev: Device struct. diff --git a/gasket_core.h b/gasket_core.h index 4501344..e51acad 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -699,11 +699,6 @@ const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev); /* Get the device structure for a given device. */ struct device *gasket_get_device(struct gasket_dev *dev); -/* Helper function, Synchronous waits on a given set of bits. */ -int gasket_wait_sync( - struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, - u64 timeout_ns); - /* Helper function, Asynchronous waits on a given set of bits. */ int gasket_wait_with_reschedule( struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, From ef0319c4c4c616851041476db6dd21ead427f1e7 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:42 -0700 Subject: [PATCH 025/209] staging: gasket: fix typo in apex_enter_reset Fix typo in log message. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apex_driver.c b/apex_driver.c index cca4cf4..a31937d 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -488,7 +488,7 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1, APEX_RESET_DELAY, APEX_RESET_RETRY)) { gasket_log_error(gasket_dev, - "DMAs did not quiece within timeout (%d ms)", + "DMAs did not quiesce within timeout (%d ms)", APEX_RESET_RETRY * APEX_RESET_DELAY); return -EINVAL; } From cc9c05a0c8ded1ea22ce845b7fd7fa6763e81d03 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:43 -0700 Subject: [PATCH 026/209] staging: gasket: fix typo in gasket_core.h comments Grammar fixup in gasket_core.h comments describing struct gasket_interrupt_desc. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.h b/gasket_core.h index e51acad..94a5537 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -56,7 +56,7 @@ enum gasket_interrupt_type { /* Used to describe a Gasket interrupt. Contains an interrupt index, a register, * and packing data for that interrupt. The register and packing data - * fields is relevant only for PCI_MSIX interrupt type and can be + * fields are relevant only for PCI_MSIX interrupt type and can be * set to 0 for everything else. */ struct gasket_interrupt_desc { From 15638304588daa814aa511e344c0c4b58d2f901c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:44 -0700 Subject: [PATCH 027/209] staging: gasket: whitespace fix in gasket_page_table_init Tab replaced with space. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index dcd52e1..36a560c 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -347,7 +347,7 @@ int gasket_page_table_init( pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_SIMPLE) { pg_tbl->num_simple_entries = total_entries; pg_tbl->num_extended_entries = 0; - pg_tbl->extended_flag = 1ull << page_table_config->extended_bit; + pg_tbl->extended_flag = 1ull << page_table_config->extended_bit; } else { pg_tbl->num_simple_entries = 0; pg_tbl->num_extended_entries = total_entries; From dc5522d1dc6c05d09f23bf54619c80c048da94fb Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:45 -0700 Subject: [PATCH 028/209] staging: gasket: remove driver registration on class creation failure If class_create() fails, remove the gasket driver from the global registration table. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index b69b630..cbadab7 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -335,7 +335,8 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) if (IS_ERR_OR_NULL(internal->class)) { gasket_nodev_error("Cannot register %s class [ret=%ld]", driver_desc->name, PTR_ERR(internal->class)); - return PTR_ERR(internal->class); + ret = PTR_ERR(internal->class); + goto unregister_gasket_driver; } /* @@ -369,6 +370,7 @@ fail2: fail1: class_destroy(internal->class); +unregister_gasket_driver: g_descs[desc_idx].driver_desc = NULL; return ret; } From d57203b5169564a6b40c93f67eeaa90a4c676fb6 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:46 -0700 Subject: [PATCH 029/209] staging: gasket: hold mutex on gasket driver unregistration Take the global mutex on driver unregistration updates for proper ordering of updates and consistent access procedures. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gasket_core.c b/gasket_core.c index cbadab7..2ff3286 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -371,7 +371,9 @@ fail1: class_destroy(internal->class); unregister_gasket_driver: + mutex_lock(&g_mutex); g_descs[desc_idx].driver_desc = NULL; + mutex_unlock(&g_mutex); return ret; } EXPORT_SYMBOL(gasket_register_device); @@ -408,7 +410,9 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) class_destroy(internal_desc->class); /* Finally, effectively "remove" the driver. */ + mutex_lock(&g_mutex); g_descs[desc_idx].driver_desc = NULL; + mutex_unlock(&g_mutex); gasket_nodev_info("removed %s driver", driver_desc->name); } From a7546cc34ca630a54dab6c6d6ab039235b565119 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:47 -0700 Subject: [PATCH 030/209] staging: gasket: Return EBUSY on mapping create when already in use gasket_sysfs_create_mapping() return EBUSY if sysfs mapping already in use, as a more appropriate error code than the current return of EINVAL, which would indicate invalid parameters. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index e3d7706..dd4d3aa 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -194,7 +194,7 @@ int gasket_sysfs_create_mapping( "0x%p.", device); put_mapping(mapping); mutex_unlock(&function_mutex); - return -EINVAL; + return -EBUSY; } /* Find the first empty entry in the array. */ From 04f5568b570e3a10d086e6400d593a1f76a07cd8 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:48 -0700 Subject: [PATCH 031/209] staging: gasket: Remove stale pointers on error allocating attr array If gasket_sysfs_create_mapping() hits errors allocating the attribute array, remove stale pointers to device info from the mapping object. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index dd4d3aa..1c5f750 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -225,6 +225,8 @@ int gasket_sysfs_create_mapping( mapping->attribute_count = 0; if (!mapping->attributes) { gasket_nodev_error("Unable to allocate sysfs attribute array."); + mapping->device = NULL; + mapping->gasket_dev = NULL; mutex_unlock(&mapping->mutex); mutex_unlock(&function_mutex); return -ENOMEM; From dd69b6031296a93636225326d7872674b74ed75f Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:49 -0700 Subject: [PATCH 032/209] staging: gasket: convert gasket_mmap_has_permissions to bool return gasket_mmap_has_permissions() should return a boolean value. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 2ff3286..248d717 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1241,19 +1241,19 @@ static int gasket_release(struct inode *inode, struct file *file) * that the provided descriptor/range is of adequate size to hold the range to * be mapped. */ -static int gasket_mmap_has_permissions( +static bool gasket_mmap_has_permissions( struct gasket_dev *gasket_dev, struct vm_area_struct *vma, int bar_permissions) { int requested_permissions; /* Always allow sysadmin to access. */ if (capable(CAP_SYS_ADMIN)) - return 1; + return true; /* Never allow non-sysadmins to access to a dead device. */ if (gasket_dev->status != GASKET_STATUS_ALIVE) { gasket_log_info(gasket_dev, "Device is dead."); - return 0; + return false; } /* Make sure that no wrong flags are set. */ @@ -1265,7 +1265,7 @@ static int gasket_mmap_has_permissions( "Attempting to map a region with requested permissions " "0x%x, but region has permissions 0x%x.", requested_permissions, bar_permissions); - return 0; + return false; } /* Do not allow a non-owner to write. */ @@ -1275,10 +1275,10 @@ static int gasket_mmap_has_permissions( gasket_dev, "Attempting to mmap a region for write without owning " "device."); - return 0; + return false; } - return 1; + return true; } /* From c850f2c77efcf74cfe8458d177bb989566429fb3 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:50 -0700 Subject: [PATCH 033/209] staging: gasket: fix gasket_wait_with_reschedule timeout return code Return -ETIMEDOUT, not -EINVAL, on timeout, including callers. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 8 ++++---- gasket_core.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index a31937d..3a83c3d 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -490,7 +490,7 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) gasket_log_error(gasket_dev, "DMAs did not quiesce within timeout (%d ms)", APEX_RESET_RETRY * APEX_RESET_DELAY); - return -EINVAL; + return -ETIMEDOUT; } /* - Enable GCB reset (0x1 to rg_rst_gcb) */ @@ -513,7 +513,7 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) gasket_dev, "RAM did not shut down within timeout (%d ms)", APEX_RESET_RETRY * APEX_RESET_DELAY); - return -EINVAL; + return -ETIMEDOUT; } return 0; @@ -562,7 +562,7 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) gasket_dev, "RAM did not enable within timeout (%d ms)", APEX_RESET_RETRY * APEX_RESET_DELAY); - return -EINVAL; + return -ETIMEDOUT; } /* - Wait for Reset complete. */ @@ -574,7 +574,7 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) gasket_dev, "GCB did not leave reset within timeout (%d ms)", APEX_RESET_RETRY * APEX_RESET_DELAY); - return -EINVAL; + return -ETIMEDOUT; } if (!allow_hw_clock_gating) { diff --git a/gasket_core.c b/gasket_core.c index 248d717..8035662 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -2106,7 +2106,7 @@ int gasket_wait_with_reschedule( "%s timeout: reg %llx timeout (%llu ms)", __func__, offset, max_retries * delay_ms); - return -EINVAL; + return -ETIMEDOUT; } return 0; } From 11eb4eb149c3f87ba107b8c91fd0c7ba2bcfa1fe Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:51 -0700 Subject: [PATCH 034/209] staging: gasket: gasket_wait_with_reschedule use msleep Replace schedule_timeout() call with msleep() for simplicity. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 8035662..4425435 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -14,6 +14,7 @@ #include "gasket_page_table.h" #include "gasket_sysfs.h" +#include #include #include #include @@ -2097,7 +2098,7 @@ int gasket_wait_with_reschedule( tmp = gasket_dev_read_64(gasket_dev, bar, offset); if ((tmp & mask) == val) break; - schedule_timeout(msecs_to_jiffies(delay_ms)); + msleep(delay_ms); retries++; } if (retries == max_retries) { From bb0f4d4a3fd932dacdd24290492772ecda8d67cd Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:52 -0700 Subject: [PATCH 035/209] staging: gasket: gasket_wait_with_reschedule simplify logic gasket_wait_with_reschedule() is a little more clear if we just return directly when the waited-for condition is hit. This also allows the following condition check to be removed and identation of the conditionally-executed code to be reduced. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 4425435..5ae3d44 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -2097,18 +2097,12 @@ int gasket_wait_with_reschedule( while (retries < max_retries) { tmp = gasket_dev_read_64(gasket_dev, bar, offset); if ((tmp & mask) == val) - break; + return 0; msleep(delay_ms); retries++; } - if (retries == max_retries) { - gasket_log_error( - gasket_dev, - "%s timeout: reg %llx timeout (%llu ms)", - __func__, - offset, max_retries * delay_ms); - return -ETIMEDOUT; - } - return 0; + gasket_log_error(gasket_dev, "%s timeout: reg %llx timeout (%llu ms)", + __func__, offset, max_retries * delay_ms); + return -ETIMEDOUT; } EXPORT_SYMBOL(gasket_wait_with_reschedule); From d7ea69e8a43cc4f7667123e0b877446de9bed210 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:53 -0700 Subject: [PATCH 036/209] staging: gasket: gasket_wait_with_reschedule use 32 bits of retry count Don't need a 64-bit retry counter. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ++-- gasket_core.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 5ae3d44..94e64b9 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -2089,9 +2089,9 @@ struct device *gasket_get_device(struct gasket_dev *dev) **/ int gasket_wait_with_reschedule( struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, - u64 max_retries, u64 delay_ms) + uint max_retries, u64 delay_ms) { - u64 retries = 0; + uint retries = 0; u64 tmp; while (retries < max_retries) { diff --git a/gasket_core.h b/gasket_core.h index 94a5537..50ad0c8 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -702,6 +702,6 @@ struct device *gasket_get_device(struct gasket_dev *dev); /* Helper function, Asynchronous waits on a given set of bits. */ int gasket_wait_with_reschedule( struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, - u64 max_retries, u64 delay_ms); + uint max_retries, u64 delay_ms); #endif /* __GASKET_CORE_H__ */ From e7cddbb9bd93201e384180e1d183bb392425c618 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:54 -0700 Subject: [PATCH 037/209] staging: gasket: bail out of reset sequence on device callback error If device reset callback returns an error, error out at the gasket level. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 94e64b9..bc3662e 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1875,9 +1875,11 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) /* Perform a device reset of the requested type. */ ret = driver_desc->device_reset_cb(gasket_dev, reset_type); - if (ret) + if (ret) { gasket_log_error( gasket_dev, "Device reset cb returned %d.", ret); + return ret; + } /* Reinitialize the page tables and interrupt framework. */ for (i = 0; i < driver_desc->num_page_tables; ++i) From 883c9a9fa98a12bade0cd0c2cdfa0e52da42bcf3 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 17 Jul 2018 13:56:55 -0700 Subject: [PATCH 038/209] staging: gasket: drop gasket_cdev_get_info, use container_of Remove gasket_cdev_get_info(), use container_of() directly instead, drop unnecessary NULL checks. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index bc3662e..0d5ba73 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -233,18 +233,6 @@ static inline int gasket_check_and_invoke_callback_nolock( return ret; } -/* - * Retrieve device-specific data via cdev pointer. - * @cdev_ptr: Character device pointer associated with the device. - * - * This function returns the pointer to the device-specific data allocated in - * add_dev_cb for the device associated with cdev_ptr. - */ -static struct gasket_cdev_info *gasket_cdev_get_info(struct cdev *cdev_ptr) -{ - return container_of(cdev_ptr, struct gasket_cdev_info, cdev); -} - /* * Returns nonzero if the gasket_cdev_info is owned by the current thread group * ID. @@ -1095,12 +1083,9 @@ static int gasket_open(struct inode *inode, struct file *filp) const struct gasket_driver_desc *driver_desc; struct gasket_ownership *ownership; char task_name[TASK_COMM_LEN]; - struct gasket_cdev_info *dev_info = gasket_cdev_get_info(inode->i_cdev); + struct gasket_cdev_info *dev_info = + container_of(inode->i_cdev, struct gasket_cdev_info, cdev); - if (!dev_info) { - gasket_nodev_error("Unable to retrieve device data"); - return -EINVAL; - } gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; ownership = &dev_info->ownership; @@ -1182,11 +1167,8 @@ static int gasket_release(struct inode *inode, struct file *file) const struct gasket_driver_desc *driver_desc; char task_name[TASK_COMM_LEN]; struct gasket_cdev_info *dev_info = - (struct gasket_cdev_info *)gasket_cdev_get_info(inode->i_cdev); - if (!dev_info) { - gasket_nodev_error("Unable to retrieve device data"); - return -EINVAL; - } + container_of(inode->i_cdev, struct gasket_cdev_info, cdev); + gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; ownership = &dev_info->ownership; From d737e80b17222c46fb4587220bc8eb38d9c652e6 Mon Sep 17 00:00:00 2001 From: Alex Van Damme Date: Mon, 23 Jul 2018 15:52:12 -0700 Subject: [PATCH 039/209] Enable Apex on ARM64, fix compile issue - Change from X86 only to X86 or ARM64 in Kconfig - refcount_read doesn't exist in 4.11, so use atomic_read instead. This is the same call that refcount_read in 4.12+ would make. Change-Id: I48c97dd8c14136dcccaa8378b3a931a7872e1289 --- Kconfig | 2 +- gasket_sysfs.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Kconfig b/Kconfig index c836389..970e299 100644 --- a/Kconfig +++ b/Kconfig @@ -2,7 +2,7 @@ menu "Gasket devices" config STAGING_GASKET_FRAMEWORK tristate "Gasket framework" - depends on PCI && X86_64 + depends on PCI && (X86_64 || ARM64) help This framework supports Gasket-compatible devices, such as Apex. It is required for any of the following module(s). diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 1c5f750..197bf53 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -108,7 +108,7 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) } mutex_lock(&mapping->mutex); - if (refcount_read(&mapping->refcount.refcount) == 0) + if (atomic_read(&mapping->refcount.refcount) == 0) gasket_nodev_error("Refcount is already 0!"); if (kref_put(&mapping->refcount, release_entry)) { gasket_nodev_info("Removing Gasket sysfs mapping, device %s", From 237454b288960eb5120ca26529084a59a6b96b9a Mon Sep 17 00:00:00 2001 From: Alex Van Damme Date: Mon, 23 Jul 2018 16:06:02 -0700 Subject: [PATCH 040/209] Apex performance ioctl Change-Id: Id3ae759f748b3f8c6f6a5320a5088c43fad0e812 --- apex.h | 15 +++++++++++ apex_driver.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/apex.h b/apex.h index 4ef2641..7279589 100644 --- a/apex.h +++ b/apex.h @@ -77,6 +77,19 @@ struct apex_gate_clock_ioctl { u64 force_idle; }; +/* Performance expectation ioctl. */ +enum apex_performance_expectation { + APEX_PERFORMANCE_LOW = 0, + APEX_PERFORMANCE_MED = 1, + APEX_PERFORMANCE_HIGH = 2, + APEX_PERFORMANCE_MAX = 3, +}; + +struct apex_performance_expectation_ioctl { + /* Expected performance from apex. */ + uint32_t performance; +}; + /* Base number for all Apex-common IOCTLs */ #define APEX_IOCTL_BASE 0x7F @@ -84,4 +97,6 @@ struct apex_gate_clock_ioctl { #define APEX_IOCTL_GATE_CLOCK \ _IOW(APEX_IOCTL_BASE, 0, struct apex_gate_clock_ioctl) +#define APEX_IOCTL_PERFORMANCE_EXPECTATION _IOW(APEX_IOCTL_BASE, 1, struct apex_performance_expectation_ioctl) + #endif /* __APEX_H__ */ diff --git a/apex_driver.c b/apex_driver.c index 3a83c3d..b25f763 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -146,6 +146,8 @@ static long apex_ioctl(struct file *file, uint cmd, ulong arg); static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg); +static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulong arg); + static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type); static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type); @@ -652,6 +654,8 @@ static long apex_ioctl(struct file *filp, uint cmd, ulong arg) switch (cmd) { case APEX_IOCTL_GATE_CLOCK: return apex_clock_gating(gasket_dev, arg); + case APEX_IOCTL_PERFORMANCE_EXPECTATION: + return apex_set_performance_expectation(gasket_dev, arg); default: return -ENOTTY; /* unknown command */ } @@ -697,6 +701,72 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg) return 0; } +/* + * apex_set_performance_expectation: Adjust clock rates for Apex. + * @gasket_dev: device pointer. + * @arg: User ioctl arg, in this case to an apex_performance_expectation_ioctl struct. + */ +static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulong arg) +{ + struct apex_performance_expectation_ioctl ibuf; + uint32_t rg_gcb_clk_div = 0; + uint32_t rg_axi_clk_125m = 0; + const int AXI_CLK_125M_SHIFT = 2; + const int MCU_CLK_250M_SHIFT = 3; + + // 8051 clock is always 250 MHz for PCIe, as it's not used at all. + const uint32_t rg_8051_clk_250m = 1; + + if (bypass_top_level) + return 0; + + if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf))) + return -EFAULT; + + switch (ibuf.performance) { + case APEX_PERFORMANCE_LOW: + // - GCB clock: 62.5 MHz + // - AXI clock: 125 MHz + rg_gcb_clk_div = 3; + rg_axi_clk_125m = 0; + break; + + case APEX_PERFORMANCE_MED: + // - GCB clock: 125 MHz + // - AXI clock: 125 MHz + rg_gcb_clk_div = 2; + rg_axi_clk_125m = 0; + break; + + case APEX_PERFORMANCE_HIGH: + // - GCB clock: 250 MHz + // - AXI clock: 125 MHz + rg_gcb_clk_div = 1; + rg_axi_clk_125m = 0; + break; + + case APEX_PERFORMANCE_MAX: + // - GCB clock: 500 MHz + // - AXI clock: 125 MHz + rg_gcb_clk_div = 0; + rg_axi_clk_125m = 0; + break; + + default: + return -EINVAL; + } + + /* + * Set clock rates for GCB, AXI, and 8051: + */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, + (rg_gcb_clk_div | (rg_axi_clk_125m << AXI_CLK_125M_SHIFT) | (rg_8051_clk_250m << MCU_CLK_250M_SHIFT)), + /*mask_width=*/4, /*mask_shift=*/28); + + return 0; +} + /* * Display driver sysfs entries. * @device: Kernel device structure. From 378d57f04e8d37b0ee5b428df5a64a301717c263 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Jul 2018 17:27:43 -0700 Subject: [PATCH 041/209] Keep apex in reset mode unless it's used. Change-Id: I56eb88a1a138290f6939a19e8b973fb78d2e3c40 --- apex_driver.c | 2 +- gasket_core.c | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apex_driver.c b/apex_driver.c index b25f763..130b1b8 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -311,7 +311,7 @@ module_init(apex_init); module_exit(apex_exit); /* Allows device to enter power save upon driver close(). */ -static int allow_power_save; +static int allow_power_save = 1; /* Allows SW based clock gating. */ static int allow_sw_clock_gating; diff --git a/gasket_core.c b/gasket_core.c index 0d5ba73..c1c9276 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -620,6 +620,16 @@ static int gasket_pci_probe( goto fail5; } + if (driver_desc->device_close_cb) { + /* Perform a device cleanup, so as to place it in Power Save + * Mode. */ + ret = driver_desc->device_close_cb(gasket_dev); + if (ret) { + gasket_log_error( + gasket_dev, "Device cleanup cb returned %d.", ret); + } + } + return 0; fail5: From c29bd5bbb6e1ef9fabe019d9e3887ad3f5182fd4 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:02 -0700 Subject: [PATCH 042/209] staging: gasket: gasket_enable_dev remove unnecessary variable Remove unnecessary variable, pass constant param instead. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index c1c9276..c64330f 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -908,7 +908,6 @@ static int gasket_enable_dev( { int tbl_idx; int ret; - bool has_dma_ops; struct device *ddev; const struct gasket_driver_desc *driver_desc = internal_desc->driver_desc; @@ -927,8 +926,6 @@ static int gasket_enable_dev( return ret; } - has_dma_ops = true; - for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { gasket_log_debug( gasket_dev, "Initializing page table %d.", tbl_idx); @@ -946,7 +943,7 @@ static int gasket_enable_dev( &gasket_dev->bar_data[ driver_desc->page_table_bar_index], &driver_desc->page_table_configs[tbl_idx], - ddev, gasket_dev->pci_dev, has_dma_ops); + ddev, gasket_dev->pci_dev, true); if (ret) { gasket_log_error( gasket_dev, From aca27c50a99e994ce91d6d98b13bc5f1d65ddc40 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:03 -0700 Subject: [PATCH 043/209] staging: gasket: remove code for no physical device gasket_enable_dev code for enabling a gasket device with no physical PCI device registered shouldn't be necessary. Reported-by: Greg Kroah-Hartman Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index c64330f..de98ee4 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -908,7 +908,6 @@ static int gasket_enable_dev( { int tbl_idx; int ret; - struct device *ddev; const struct gasket_driver_desc *driver_desc = internal_desc->driver_desc; @@ -929,21 +928,12 @@ static int gasket_enable_dev( for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { gasket_log_debug( gasket_dev, "Initializing page table %d.", tbl_idx); - if (gasket_dev->pci_dev) { - ddev = &gasket_dev->pci_dev->dev; - } else { - gasket_log_error( - gasket_dev, - "%s with no physical device!!", __func__); - WARN_ON(1); - ddev = NULL; - } ret = gasket_page_table_init( &gasket_dev->page_table[tbl_idx], &gasket_dev->bar_data[ driver_desc->page_table_bar_index], &driver_desc->page_table_configs[tbl_idx], - ddev, gasket_dev->pci_dev, true); + &gasket_dev->pci_dev->dev, gasket_dev->pci_dev, true); if (ret) { gasket_log_error( gasket_dev, From 48ad480759672743b08637ecb8aacca26cddf681 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:04 -0700 Subject: [PATCH 044/209] staging: gasket: fix class create bug handling class_create() never returns NULL, and this driver should never return PTR_ERR(NULL) anyway. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Reviewed-by: Dmitry Torokhov Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index de98ee4..7360468 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -321,7 +321,7 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) internal->class = class_create(driver_desc->module, driver_desc->name); - if (IS_ERR_OR_NULL(internal->class)) { + if (IS_ERR(internal->class)) { gasket_nodev_error("Cannot register %s class [ret=%ld]", driver_desc->name, PTR_ERR(internal->class)); ret = PTR_ERR(internal->class); From 087af2746125fb2070d8648a484b1a2f2541be7b Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:05 -0700 Subject: [PATCH 045/209] staging: gasket: remove unnecessary code in coherent allocator Remove extraneous statement in gasket_config_coherent_allocator() Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 0c2f85c..d0142ed 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -420,10 +420,8 @@ static int gasket_config_coherent_allocator( if (ibuf.page_table_index >= gasket_dev->num_page_tables) return -EFAULT; - if (ibuf.size > PAGE_SIZE * MAX_NUM_COHERENT_PAGES) { - ibuf.size = PAGE_SIZE * MAX_NUM_COHERENT_PAGES; + if (ibuf.size > PAGE_SIZE * MAX_NUM_COHERENT_PAGES) return -ENOMEM; - } if (ibuf.enable == 0) { ret = gasket_free_coherent_memory( From a666b28f2c74f3e5336cbc21a217024dac2e729c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:06 -0700 Subject: [PATCH 046/209] staging: gasket: don't treat no device reset callback as an error It is not an error for a device to not have a reset callback registered. Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 7360468..7b912f1 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1846,11 +1846,8 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) const struct gasket_driver_desc *driver_desc; driver_desc = gasket_dev->internal_desc->driver_desc; - if (!driver_desc->device_reset_cb) { - gasket_log_error( - gasket_dev, "No device reset callback was registered."); - return -EINVAL; - } + if (!driver_desc->device_reset_cb) + return 0; /* Perform a device reset of the requested type. */ ret = driver_desc->device_reset_cb(gasket_dev, reset_type); From 21ea6458c216abab02c9b30a5209c018e7c173a1 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:07 -0700 Subject: [PATCH 047/209] staging: gasket: gasket_mmap return error instead of valid BAR index When offset to be mapped matches both a BAR region and a coherent mapped region return an error as intended, not the BAR index. Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 7b912f1..43c63da 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1637,7 +1637,7 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) "0x%lx", raw_offset); trace_gasket_mmap_exit(bar_index); - return bar_index; + return -EINVAL; } vma->vm_private_data = gasket_dev; From 5f5275b6c89610dbfe18bc2b034390de363ecd70 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:08 -0700 Subject: [PATCH 048/209] staging: gasket: apex_clock_gating simplify logic, reduce indentation Collapse together two checks and return immediately, avoid conditional indentation for most of function code. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 130b1b8..41679fd 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -670,33 +670,30 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg) { struct apex_gate_clock_ioctl ibuf; - if (bypass_top_level) + if (bypass_top_level || !allow_sw_clock_gating) return 0; - if (allow_sw_clock_gating) { - if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf))) - return -EFAULT; + if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf))) + return -EFAULT; - gasket_log_error( - gasket_dev, "%s %llu", __func__, ibuf.enable); + gasket_log_error(gasket_dev, "%s %llu", __func__, ibuf.enable); - if (ibuf.enable) { - /* Quiesce AXI, gate GCB clock. */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, 16); - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, 2, 18); - } else { - /* Un-gate GCB clock, un-quiesce AXI. */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, 2, 18); - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, 16); - } + if (ibuf.enable) { + /* Quiesce AXI, gate GCB clock. */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, 16); + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, 2, 18); + } else { + /* Un-gate GCB clock, un-quiesce AXI. */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, 2, 18); + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, 16); } return 0; } From b710f906c50990c103e5219021681f8b373a6f8a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:09 -0700 Subject: [PATCH 049/209] staging: gasket: gasket page table functions use bool return type Convert from int to bool return type for gasket page table functions that return values used as booleans. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 58 ++++++++++++++++++++++----------------------- gasket_page_table.h | 8 +++---- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 36a560c..2a27db6 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -262,16 +262,16 @@ static void gasket_perform_unmapping( static void gasket_free_extended_subtable( struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, u64 __iomem *att_reg); -static int gasket_release_page(struct page *page); +static bool gasket_release_page(struct page *page); /* Other/utility declarations */ -static inline int gasket_addr_is_simple( +static inline bool gasket_addr_is_simple( struct gasket_page_table *pg_tbl, ulong addr); -static int gasket_is_simple_dev_addr_bad( +static bool gasket_is_simple_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); -static int gasket_is_extended_dev_addr_bad( +static bool gasket_is_extended_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); -static int gasket_is_pte_range_free( +static bool gasket_is_pte_range_free( struct gasket_page_table_entry *pte, uint num_entries); static void gasket_page_table_garbage_collect_nolock( struct gasket_page_table *pg_tbl); @@ -558,7 +558,7 @@ fail: } /* See gasket_page_table.h for description. */ -int gasket_page_table_are_addrs_bad( +bool gasket_page_table_are_addrs_bad( struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, ulong bytes) { @@ -567,7 +567,7 @@ int gasket_page_table_are_addrs_bad( pg_tbl, "host mapping address 0x%lx must be page aligned", host_addr); - return 1; + return true; } return gasket_page_table_is_dev_addr_bad(pg_tbl, dev_addr, bytes); @@ -575,7 +575,7 @@ int gasket_page_table_are_addrs_bad( EXPORT_SYMBOL(gasket_page_table_are_addrs_bad); /* See gasket_page_table.h for description. */ -int gasket_page_table_is_dev_addr_bad( +bool gasket_page_table_is_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes) { uint num_pages = bytes / PAGE_SIZE; @@ -584,7 +584,7 @@ int gasket_page_table_is_dev_addr_bad( gasket_pg_tbl_error( pg_tbl, "mapping size 0x%lX must be page aligned", bytes); - return 1; + return true; } if (num_pages == 0) { @@ -592,7 +592,7 @@ int gasket_page_table_is_dev_addr_bad( pg_tbl, "requested mapping is less than one page: %lu / %lu", bytes, PAGE_SIZE); - return 1; + return true; } if (gasket_addr_is_simple(pg_tbl, dev_addr)) @@ -1285,23 +1285,23 @@ static void gasket_free_extended_subtable( /* * Safely return a page to the OS. * @page: The page to return to the OS. - * Returns 1 if the page was released, 0 if it was + * Returns true if the page was released, false if it was * ignored. */ -static int gasket_release_page(struct page *page) +static bool gasket_release_page(struct page *page) { if (!page) - return 0; + return false; if (!PageReserved(page)) SetPageDirty(page); put_page(page); - return 1; + return true; } /* Evaluates to nonzero if the specified virtual address is simple. */ -static inline int gasket_addr_is_simple( +static inline bool gasket_addr_is_simple( struct gasket_page_table *pg_tbl, ulong addr) { return !((addr) & (pg_tbl)->extended_flag); @@ -1317,7 +1317,7 @@ static inline int gasket_addr_is_simple( * address to/from page + offset) and that the requested page range starts and * ends within the set of currently-partitioned simple pages. */ -static int gasket_is_simple_dev_addr_bad( +static bool gasket_is_simple_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) { ulong page_offset = dev_addr & (PAGE_SIZE - 1); @@ -1328,7 +1328,7 @@ static int gasket_is_simple_dev_addr_bad( pg_tbl, 1, page_index, page_offset) != dev_addr) { gasket_pg_tbl_error( pg_tbl, "address is invalid, 0x%lX", dev_addr); - return 1; + return true; } if (page_index >= pg_tbl->num_simple_entries) { @@ -1336,7 +1336,7 @@ static int gasket_is_simple_dev_addr_bad( pg_tbl, "starting slot at %lu is too large, max is < %u", page_index, pg_tbl->num_simple_entries); - return 1; + return true; } if (page_index + num_pages > pg_tbl->num_simple_entries) { @@ -1344,10 +1344,10 @@ static int gasket_is_simple_dev_addr_bad( pg_tbl, "ending slot at %lu is too large, max is <= %u", page_index + num_pages, pg_tbl->num_simple_entries); - return 1; + return true; } - return 0; + return false; } /* @@ -1359,7 +1359,7 @@ static int gasket_is_simple_dev_addr_bad( * @dev_addr: The device address to which the pages will be mapped. * @num_pages: The number of second-level/sub pages in the range to consider. */ -static int gasket_is_extended_dev_addr_bad( +static bool gasket_is_extended_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) { /* Starting byte index of dev_addr into the first mapped page */ @@ -1373,7 +1373,7 @@ static int gasket_is_extended_dev_addr_bad( if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { gasket_pg_tbl_error(pg_tbl, "device address out of bound, 0x%p", (void *)dev_addr); - return 1; + return true; } /* Find the starting sub-page index in the space of all sub-pages. */ @@ -1391,7 +1391,7 @@ static int gasket_is_extended_dev_addr_bad( pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { gasket_pg_tbl_error( pg_tbl, "address is invalid, 0x%p", (void *)dev_addr); - return 1; + return true; } if (page_lvl0_idx >= pg_tbl->num_extended_entries) { @@ -1399,7 +1399,7 @@ static int gasket_is_extended_dev_addr_bad( pg_tbl, "starting level 0 slot at %lu is too large, max is < " "%u", page_lvl0_idx, pg_tbl->num_extended_entries); - return 1; + return true; } if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) { @@ -1408,10 +1408,10 @@ static int gasket_is_extended_dev_addr_bad( "ending level 0 slot at %lu is too large, max is <= %u", page_lvl0_idx + num_lvl0_pages, pg_tbl->num_extended_entries); - return 1; + return true; } - return 0; + return false; } /* @@ -1425,17 +1425,17 @@ static int gasket_is_extended_dev_addr_bad( * * The page table mutex must be held before this call. */ -static int gasket_is_pte_range_free( +static bool gasket_is_pte_range_free( struct gasket_page_table_entry *ptes, uint num_entries) { int i; for (i = 0; i < num_entries; i++) { if (ptes[i].status != PTE_FREE) - return 0; + return false; } - return 1; + return true; } /* diff --git a/gasket_page_table.h b/gasket_page_table.h index 720ce2b..0e8afdb 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -162,9 +162,9 @@ int gasket_page_table_lookup_page( * specified by both addresses and the size are valid for mapping pages into * device memory. * - * Returns 1 if true - if the mapping is bad, 0 otherwise. + * Returns true if the mapping is bad, false otherwise. */ -int gasket_page_table_are_addrs_bad( +bool gasket_page_table_are_addrs_bad( struct gasket_page_table *page_table, ulong host_addr, ulong dev_addr, ulong bytes); @@ -178,9 +178,9 @@ int gasket_page_table_are_addrs_bad( * specified by the device address and the size is valid for mapping pages into * device memory. * - * Returns 1 if true - if the address is bad, 0 otherwise. + * Returns true if the address is bad, false otherwise. */ -int gasket_page_table_is_dev_addr_bad( +bool gasket_page_table_is_dev_addr_bad( struct gasket_page_table *page_table, ulong dev_addr, ulong bytes); /* From 35ae6935e7d8d8792f491e25dc6f990df118e087 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:10 -0700 Subject: [PATCH 050/209] staging: gasket: remove else clause after return in if clause Else after return is unnecessary and may cause static code checkers to complain. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 2a27db6..617d602 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -598,9 +598,7 @@ bool gasket_page_table_is_dev_addr_bad( if (gasket_addr_is_simple(pg_tbl, dev_addr)) return gasket_is_simple_dev_addr_bad( pg_tbl, dev_addr, num_pages); - else - return gasket_is_extended_dev_addr_bad( - pg_tbl, dev_addr, num_pages); + return gasket_is_extended_dev_addr_bad(pg_tbl, dev_addr, num_pages); } EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad); From e4c3d028296a856cc8367e394c4daa49a5b14d84 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:11 -0700 Subject: [PATCH 051/209] staging: gasket: fix comment syntax in apex.h Use kernel-style multi-line comment syntax. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apex.h b/apex.h index 7279589..c7dd71e 100644 --- a/apex.h +++ b/apex.h @@ -22,9 +22,10 @@ #define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */ -/* Addresses are 2^3=8 bytes each. */ -/* page in second level page table */ -/* holds APEX_PAGE_SIZE/8 addresses */ +/* + * Addresses are 2^3=8 bytes each. Page in second level page table holds + * APEX_PAGE_SIZE/8 addresses. + */ #define APEX_ADDR_SHIFT 3 #define APEX_LEVEL_SHIFT (APEX_PAGE_SHIFT - APEX_ADDR_SHIFT) #define APEX_LEVEL_SIZE BIT(APEX_LEVEL_SHIFT) From d99aa15f2e70b1555c98aef7b94413225f26579c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:12 -0700 Subject: [PATCH 052/209] staging: gasket: remove unnecessary parens in page table code gasket_alloc_coherent_memory() extra parentheses in statement. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 617d602..9f81161 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1639,7 +1639,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, dma_addr_t handle; void *mem; int j; - unsigned int num_pages = (size + PAGE_SIZE - 1) / (PAGE_SIZE); + unsigned int num_pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; const struct gasket_driver_desc *driver_desc = gasket_get_driver_desc(gasket_dev); From 4e7e5697243c985136ea08167a04ffd8312c33da Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:13 -0700 Subject: [PATCH 053/209] staging: gasket: gasket_mmap use PAGE_MASK gasket_mmap use PAGE_MASK, instead of performing math on PAGE_SIZE, for simplicity and clarity. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 43c63da..610d035 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1601,7 +1601,7 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) } driver_desc = gasket_dev->internal_desc->driver_desc; - if (vma->vm_start & (PAGE_SIZE - 1)) { + if (vma->vm_start & ~PAGE_MASK) { gasket_log_error( gasket_dev, "Base address not page-aligned: 0x%p\n", (void *)vma->vm_start); From c685c4ea9cffdf247523ee8dd8af90cbbcacb53e Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:14 -0700 Subject: [PATCH 054/209] staging: gasket: remove extra parens in gasket_write_mappable_regions Remove unneeded parentheses around subexpressions. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 610d035..2ad51cf 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1899,7 +1899,7 @@ static ssize_t gasket_write_mappable_regions( if (bar_desc.permissions == GASKET_NOMAP) return 0; for (i = 0; - (i < bar_desc.num_mappable_regions) && (total_written < PAGE_SIZE); + i < bar_desc.num_mappable_regions && total_written < PAGE_SIZE; i++) { min_addr = bar_desc.mappable_regions[i].start - driver_desc->legacy_mmap_address_offset; From 4984eecc6323369c8dd69ecb41898531e09073bc Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:15 -0700 Subject: [PATCH 055/209] staging: gasket: fix multi-line comment syntax in gasket_core.h Use consistent kernel-style multi-line comment syntax. Reported-by: Guenter Roeck Signed-off-by: Simon Que Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gasket_core.h b/gasket_core.h index 50ad0c8..7ea1df1 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -54,7 +54,8 @@ enum gasket_interrupt_type { PLATFORM_WIRE = 2, }; -/* Used to describe a Gasket interrupt. Contains an interrupt index, a register, +/* + * Used to describe a Gasket interrupt. Contains an interrupt index, a register, * and packing data for that interrupt. The register and packing data * fields are relevant only for PCI_MSIX interrupt type and can be * set to 0 for everything else. From c4e2ac27037e300c86f4f57251af47d9cf4b5392 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:16 -0700 Subject: [PATCH 056/209] staging: gasket: always allow root open for write Always allow root to open device for writing. Drop special-casing of ioctl permissions for root vs. owner. Convert to bool types as appropriate. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 15 ++++----------- gasket_core.c | 8 +++++--- gasket_ioctl.c | 30 ++++++++++++++---------------- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 41679fd..c1cef70 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -140,7 +140,7 @@ static int apex_reset(struct gasket_dev *gasket_dev, uint type); static int apex_get_status(struct gasket_dev *gasket_dev); -static uint apex_ioctl_check_permissions(struct file *file, uint cmd); +static bool apex_ioctl_check_permissions(struct file *file, uint cmd); static long apex_ioctl(struct file *file, uint cmd, ulong arg); @@ -627,18 +627,11 @@ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) * @file: File pointer from ioctl. * @cmd: ioctl command. * - * Returns 1 if the current user may execute this ioctl, and 0 otherwise. + * Returns true if the current user may execute this ioctl, and false otherwise. */ -static uint apex_ioctl_check_permissions(struct file *filp, uint cmd) +static bool apex_ioctl_check_permissions(struct file *filp, uint cmd) { - struct gasket_dev *gasket_dev = filp->private_data; - int root = capable(CAP_SYS_ADMIN); - int is_owner = gasket_dev->dev_info.ownership.is_owned && - current->tgid == gasket_dev->dev_info.ownership.owner; - - if (root || is_owner) - return 1; - return 0; + return !!(filp->f_mode & FMODE_WRITE); } /* diff --git a/gasket_core.c b/gasket_core.c index 2ad51cf..37f2242 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1082,6 +1082,7 @@ static int gasket_open(struct inode *inode, struct file *filp) char task_name[TASK_COMM_LEN]; struct gasket_cdev_info *dev_info = container_of(inode->i_cdev, struct gasket_cdev_info, cdev); + int is_root = capable(CAP_SYS_ADMIN); gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; @@ -1095,7 +1096,7 @@ static int gasket_open(struct inode *inode, struct file *filp) "Attempting to open with tgid %u (%s) (f_mode: 0%03o, " "fmode_write: %d is_root: %u)", current->tgid, task_name, filp->f_mode, - (filp->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN)); + (filp->f_mode & FMODE_WRITE), is_root); /* Always allow non-writing accesses. */ if (!(filp->f_mode & FMODE_WRITE)) { @@ -1109,8 +1110,9 @@ static int gasket_open(struct inode *inode, struct file *filp) gasket_dev, "Current owner open count (owning tgid %u): %d.", ownership->owner, ownership->write_open_count); - /* Opening a node owned by another TGID is an error (even root.) */ - if (ownership->is_owned && ownership->owner != current->tgid) { + /* Opening a node owned by another TGID is an error (unless root) */ + if (ownership->is_owned && ownership->owner != current->tgid && + !is_root) { gasket_log_error( gasket_dev, "Process %u is opening a node held by %u.", diff --git a/gasket_ioctl.c b/gasket_ioctl.c index d0142ed..8fd4497 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -22,7 +22,7 @@ #define trace_gasket_ioctl_config_coherent_allocator(x, ...) #endif -static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd); +static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd); static int gasket_set_event_fd(struct gasket_dev *dev, ulong arg); static int gasket_read_page_table_size( struct gasket_dev *gasket_dev, ulong arg); @@ -167,12 +167,13 @@ long gasket_is_supported_ioctl(uint cmd) * @filp: File structure pointer describing this node usage session. * @cmd: ioctl number to handle. * - * Standard permissions checker. + * Check permissions for Gasket ioctls. + * Returns true if the file opener may execute this ioctl, or false otherwise. */ -static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd) +static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) { - uint alive, root, device_owner; - fmode_t read, write; + bool alive; + bool read, write; struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; alive = (gasket_dev->status == GASKET_STATUS_ALIVE); @@ -183,36 +184,33 @@ static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd) alive, gasket_dev->status); } - root = capable(CAP_SYS_ADMIN); - read = filp->f_mode & FMODE_READ; - write = filp->f_mode & FMODE_WRITE; - device_owner = (gasket_dev->dev_info.ownership.is_owned && - current->tgid == gasket_dev->dev_info.ownership.owner); + read = !!(filp->f_mode & FMODE_READ); + write = !!(filp->f_mode & FMODE_WRITE); switch (cmd) { case GASKET_IOCTL_RESET: case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: - return root || (write && device_owner); + return write; case GASKET_IOCTL_PAGE_TABLE_SIZE: case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: case GASKET_IOCTL_NUMBER_PAGE_TABLES: - return root || read; + return read; case GASKET_IOCTL_PARTITION_PAGE_TABLE: case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: - return alive && (root || (write && device_owner)); + return alive && write; case GASKET_IOCTL_MAP_BUFFER: case GASKET_IOCTL_UNMAP_BUFFER: - return alive && (root || (write && device_owner)); + return alive && write; case GASKET_IOCTL_CLEAR_EVENTFD: case GASKET_IOCTL_SET_EVENTFD: - return alive && (root || (write && device_owner)); + return alive && write; } - return 0; /* unknown permissions */ + return false; /* unknown permissions */ } /* From 73e7d5fac54502ceb215fe5c5a2be4dde7460278 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:17 -0700 Subject: [PATCH 057/209] staging: gasket: top ioctl handler add __user annotations Add __user annotation to gasket_core top-level ioctl handling pointer arguments, for sparse checking. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 6 ++++-- gasket_core.h | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 37f2242..45ffb60 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -14,6 +14,7 @@ #include "gasket_page_table.h" #include "gasket_sysfs.h" +#include #include #include #include @@ -1791,6 +1792,7 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) { struct gasket_dev *gasket_dev; const struct gasket_driver_desc *driver_desc; + void __user *argp = (void __user *)arg; char path[256]; if (!filp) @@ -1820,14 +1822,14 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) * check_and_invoke_callback. */ if (driver_desc->ioctl_handler_cb) - return driver_desc->ioctl_handler_cb(filp, cmd, arg); + return driver_desc->ioctl_handler_cb(filp, cmd, argp); gasket_log_error( gasket_dev, "Received unknown ioctl 0x%x", cmd); return -EINVAL; } - return gasket_handle_ioctl(filp, cmd, arg); + return gasket_handle_ioctl(filp, cmd, argp); } int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type) diff --git a/gasket_core.h b/gasket_core.h index 7ea1df1..bf4ed37 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -314,9 +314,12 @@ struct gasket_dev { struct hlist_node legacy_hlist_node; }; +/* Type of the ioctl handler callback. */ +typedef long (*gasket_ioctl_handler_cb_t) + (struct file *file, uint cmd, void __user *argp); /* Type of the ioctl permissions check callback. See below. */ typedef int (*gasket_ioctl_permissions_cb_t)( - struct file *filp, uint cmd, ulong arg); + struct file *filp, uint cmd, void __user *argp); /* * Device type descriptor. @@ -550,7 +553,7 @@ struct gasket_driver_desc { * return -EINVAL. Should return an error status (either -EINVAL or * the error result of the ioctl being handled). */ - long (*ioctl_handler_cb)(struct file *filp, uint cmd, ulong arg); + gasket_ioctl_handler_cb_t ioctl_handler_cb; /* * device_status_cb: Callback to determine device health. From 53f54339d769b63f1719c0fab6ca74be29350162 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:18 -0700 Subject: [PATCH 058/209] staging: gasket: apex ioctl add __user annotations Add __user annotation to ioctl pointer argument, for sparse checking. Change-Id: I677aa0a2eed9a34b799273e0a3a7cb45ced14c8e Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index c1cef70..c95071e 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -5,6 +5,7 @@ * Copyright (C) 2018 Google, Inc. */ +#include #include #include #include @@ -142,9 +143,10 @@ static int apex_get_status(struct gasket_dev *gasket_dev); static bool apex_ioctl_check_permissions(struct file *file, uint cmd); -static long apex_ioctl(struct file *file, uint cmd, ulong arg); +static long apex_ioctl(struct file *file, uint cmd, void __user *argp); -static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg); +static long apex_clock_gating(struct gasket_dev *gasket_dev, + struct apex_gate_clock_ioctl __user *argp); static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulong arg); @@ -637,7 +639,7 @@ static bool apex_ioctl_check_permissions(struct file *filp, uint cmd) /* * Apex-specific ioctl handler. */ -static long apex_ioctl(struct file *filp, uint cmd, ulong arg) +static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) { struct gasket_dev *gasket_dev = filp->private_data; @@ -646,9 +648,9 @@ static long apex_ioctl(struct file *filp, uint cmd, ulong arg) switch (cmd) { case APEX_IOCTL_GATE_CLOCK: - return apex_clock_gating(gasket_dev, arg); + return apex_clock_gating(gasket_dev, argp); case APEX_IOCTL_PERFORMANCE_EXPECTATION: - return apex_set_performance_expectation(gasket_dev, arg); + return apex_set_performance_expectation(gasket_dev, argp); default: return -ENOTTY; /* unknown command */ } @@ -657,16 +659,17 @@ static long apex_ioctl(struct file *filp, uint cmd, ulong arg) /* * Gates or un-gates Apex clock. * @gasket_dev: Gasket device pointer. - * @arg: User ioctl arg, in this case to a apex_gate_clock_ioctl struct. + * @argp: User ioctl arg, pointer to a apex_gate_clock_ioctl struct. */ -static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg) +static long apex_clock_gating(struct gasket_dev *gasket_dev, + struct apex_gate_clock_ioctl __user *argp) { struct apex_gate_clock_ioctl ibuf; if (bypass_top_level || !allow_sw_clock_gating) return 0; - if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf))) + if (copy_from_user(&ibuf, argp, sizeof(ibuf))) return -EFAULT; gasket_log_error(gasket_dev, "%s %llu", __func__, ibuf.enable); From 342c986d283eac3e0d9fadff0c2f842b3049eb1c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:19 -0700 Subject: [PATCH 059/209] staging: gasket: common ioctl dispatcher add __user annotations Add __user annotation to gasket core common ioctl pointer arguments for sparse checking. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 8 +++++--- gasket_ioctl.h | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 8fd4497..998d0e2 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -7,6 +7,7 @@ #include "gasket_interrupt.h" #include "gasket_logging.h" #include "gasket_page_table.h" +#include #include #include @@ -39,13 +40,14 @@ static int gasket_config_coherent_allocator( * standard ioctl dispatch function. * @filp: File structure pointer describing this node usage session. * @cmd: ioctl number to handle. - * @arg: ioctl-specific data pointer. + * @argp: ioctl-specific data pointer. * * Standard ioctl dispatcher; forwards operations to individual handlers. */ -long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg) +long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) { struct gasket_dev *gasket_dev; + unsigned long arg = (unsigned long)argp; int retval; gasket_dev = (struct gasket_dev *)filp->private_data; @@ -53,7 +55,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg) if (gasket_get_ioctl_permissions_cb(gasket_dev)) { retval = gasket_get_ioctl_permissions_cb(gasket_dev)( - filp, cmd, arg); + filp, cmd, argp); if (retval < 0) { trace_gasket_ioctl_exit(-EPERM); return retval; diff --git a/gasket_ioctl.h b/gasket_ioctl.h index 461fab2..51f468c 100644 --- a/gasket_ioctl.h +++ b/gasket_ioctl.h @@ -5,6 +5,8 @@ #include "gasket_core.h" +#include + /* * Handle Gasket common ioctls. * @filp: Pointer to the ioctl's file. @@ -13,7 +15,7 @@ * * Returns 0 on success and nonzero on failure. */ -long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg); +long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp); /* * Determines if an ioctl is part of the standard Gasket framework. From 0a9d6ccbbfaa48a0a9d41247f3f30c02da747736 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 19 Jul 2018 20:49:20 -0700 Subject: [PATCH 060/209] staging: gasket: common ioctls add __user annotations Add __user annotation to gasket common ioctl pointer arguments for sparse checking. Reported-by: Dmitry Torokhov Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 102 ++++++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 998d0e2..2e2c9b9 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -24,17 +24,24 @@ #endif static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd); -static int gasket_set_event_fd(struct gasket_dev *dev, ulong arg); +static int gasket_set_event_fd(struct gasket_dev *dev, + struct gasket_interrupt_eventfd __user *argp); static int gasket_read_page_table_size( - struct gasket_dev *gasket_dev, ulong arg); + struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp); static int gasket_read_simple_page_table_size( - struct gasket_dev *gasket_dev, ulong arg); + struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp); static int gasket_partition_page_table( - struct gasket_dev *gasket_dev, ulong arg); -static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg); -static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg); + struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp); +static int gasket_map_buffers(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp); +static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp); static int gasket_config_coherent_allocator( - struct gasket_dev *gasket_dev, ulong arg); + struct gasket_dev *gasket_dev, + struct gasket_coherent_alloc_config_ioctl __user *argp); /* * standard ioctl dispatch function. @@ -80,7 +87,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) retval = gasket_reset(gasket_dev, arg); break; case GASKET_IOCTL_SET_EVENTFD: - retval = gasket_set_event_fd(gasket_dev, arg); + retval = gasket_set_event_fd(gasket_dev, argp); break; case GASKET_IOCTL_CLEAR_EVENTFD: trace_gasket_ioctl_integer_data(arg); @@ -89,31 +96,30 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) break; case GASKET_IOCTL_PARTITION_PAGE_TABLE: trace_gasket_ioctl_integer_data(arg); - retval = gasket_partition_page_table(gasket_dev, arg); + retval = gasket_partition_page_table(gasket_dev, argp); break; case GASKET_IOCTL_NUMBER_PAGE_TABLES: trace_gasket_ioctl_integer_data(gasket_dev->num_page_tables); - if (copy_to_user((void __user *)arg, - &gasket_dev->num_page_tables, + if (copy_to_user(argp, &gasket_dev->num_page_tables, sizeof(uint64_t))) retval = -EFAULT; else retval = 0; break; case GASKET_IOCTL_PAGE_TABLE_SIZE: - retval = gasket_read_page_table_size(gasket_dev, arg); + retval = gasket_read_page_table_size(gasket_dev, argp); break; case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: - retval = gasket_read_simple_page_table_size(gasket_dev, arg); + retval = gasket_read_simple_page_table_size(gasket_dev, argp); break; case GASKET_IOCTL_MAP_BUFFER: - retval = gasket_map_buffers(gasket_dev, arg); + retval = gasket_map_buffers(gasket_dev, argp); break; case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: - retval = gasket_config_coherent_allocator(gasket_dev, arg); + retval = gasket_config_coherent_allocator(gasket_dev, argp); break; case GASKET_IOCTL_UNMAP_BUFFER: - retval = gasket_unmap_buffers(gasket_dev, arg); + retval = gasket_unmap_buffers(gasket_dev, argp); break; case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: /* Clear interrupt counts doesn't take an arg, so use 0. */ @@ -218,16 +224,15 @@ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) /* * Associate an eventfd with an interrupt. * @gasket_dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to gasket_interrupt_eventfd struct in userspace. + * @argp: Pointer to gasket_interrupt_eventfd struct in userspace. */ -static int gasket_set_event_fd(struct gasket_dev *gasket_dev, ulong arg) +static int gasket_set_event_fd(struct gasket_dev *gasket_dev, + struct gasket_interrupt_eventfd __user *argp) { struct gasket_interrupt_eventfd die; - if (copy_from_user(&die, (void __user *)arg, - sizeof(struct gasket_interrupt_eventfd))) { + if (copy_from_user(&die, argp, sizeof(struct gasket_interrupt_eventfd))) return -EFAULT; - } trace_gasket_ioctl_eventfd_data(die.interrupt, die.event_fd); @@ -238,15 +243,16 @@ static int gasket_set_event_fd(struct gasket_dev *gasket_dev, ulong arg) /* * Reads the size of the page table. * @gasket_dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to gasket_page_table_ioctl struct in userspace. + * @argp: Pointer to gasket_page_table_ioctl struct in userspace. */ -static int gasket_read_page_table_size(struct gasket_dev *gasket_dev, ulong arg) +static int gasket_read_page_table_size( + struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp) { int ret = 0; struct gasket_page_table_ioctl ibuf; - if (copy_from_user(&ibuf, (void __user *)arg, - sizeof(struct gasket_page_table_ioctl))) + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; if (ibuf.page_table_index >= gasket_dev->num_page_tables) @@ -259,7 +265,7 @@ static int gasket_read_page_table_size(struct gasket_dev *gasket_dev, ulong arg) ibuf.page_table_index, ibuf.size, ibuf.host_address, ibuf.device_address); - if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf))) + if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; return ret; @@ -268,16 +274,16 @@ static int gasket_read_page_table_size(struct gasket_dev *gasket_dev, ulong arg) /* * Reads the size of the simple page table. * @gasket_dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to gasket_page_table_ioctl struct in userspace. + * @argp: Pointer to gasket_page_table_ioctl struct in userspace. */ static int gasket_read_simple_page_table_size( - struct gasket_dev *gasket_dev, ulong arg) + struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp) { int ret = 0; struct gasket_page_table_ioctl ibuf; - if (copy_from_user(&ibuf, (void __user *)arg, - sizeof(struct gasket_page_table_ioctl))) + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; if (ibuf.page_table_index >= gasket_dev->num_page_tables) @@ -290,7 +296,7 @@ static int gasket_read_simple_page_table_size( ibuf.page_table_index, ibuf.size, ibuf.host_address, ibuf.device_address); - if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf))) + if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; return ret; @@ -299,16 +305,17 @@ static int gasket_read_simple_page_table_size( /* * Sets the boundary between the simple and extended page tables. * @gasket_dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to gasket_page_table_ioctl struct in userspace. + * @argp: Pointer to gasket_page_table_ioctl struct in userspace. */ -static int gasket_partition_page_table(struct gasket_dev *gasket_dev, ulong arg) +static int gasket_partition_page_table( + struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp) { int ret; struct gasket_page_table_ioctl ibuf; uint max_page_table_size; - if (copy_from_user(&ibuf, (void __user *)arg, - sizeof(struct gasket_page_table_ioctl))) + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; trace_gasket_ioctl_page_table_data( @@ -340,14 +347,14 @@ static int gasket_partition_page_table(struct gasket_dev *gasket_dev, ulong arg) /* * Maps a userspace buffer to a device virtual address. * @gasket_dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to a gasket_page_table_ioctl struct in userspace. + * @argp: Pointer to a gasket_page_table_ioctl struct in userspace. */ -static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg) +static int gasket_map_buffers(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp) { struct gasket_page_table_ioctl ibuf; - if (copy_from_user(&ibuf, (void __user *)arg, - sizeof(struct gasket_page_table_ioctl))) + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; trace_gasket_ioctl_page_table_data( @@ -370,14 +377,14 @@ static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg) /* * Unmaps a userspace buffer from a device virtual address. * @gasket_dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to a gasket_page_table_ioctl struct in userspace. + * @argp: Pointer to a gasket_page_table_ioctl struct in userspace. */ -static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg) +static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl __user *argp) { struct gasket_page_table_ioctl ibuf; - if (copy_from_user(&ibuf, (void __user *)arg, - sizeof(struct gasket_page_table_ioctl))) + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; trace_gasket_ioctl_page_table_data( @@ -402,15 +409,16 @@ static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg) * Tell the driver to reserve structures for coherent allocation, and allocate * or free the corresponding memory. * @dev: Pointer to the current gasket_dev we're using. - * @arg: Pointer to a gasket_coherent_alloc_config_ioctl struct in userspace. + * @argp: Pointer to a gasket_coherent_alloc_config_ioctl struct in userspace. */ static int gasket_config_coherent_allocator( - struct gasket_dev *gasket_dev, ulong arg) + struct gasket_dev *gasket_dev, + struct gasket_coherent_alloc_config_ioctl __user *argp) { int ret; struct gasket_coherent_alloc_config_ioctl ibuf; - if (copy_from_user(&ibuf, (void __user *)arg, + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_coherent_alloc_config_ioctl))) return -EFAULT; @@ -432,7 +440,7 @@ static int gasket_config_coherent_allocator( gasket_dev, ibuf.size, &ibuf.dma_address, ibuf.page_table_index); } - if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf))) + if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; return ret; From eef60d21affdd70a06715efd19abf8baaa43320d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:34:54 -0700 Subject: [PATCH 061/209] staging: gasket: fix check_and_invoke_callback log param The message should be passed the callback function pointer, not the pointer to the gasket device. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 45ffb60..5e269c4 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -205,7 +205,7 @@ static inline int check_and_invoke_callback( { int ret = 0; - gasket_nodev_error("check_and_invoke_callback %p", gasket_dev); + gasket_nodev_error("check_and_invoke_callback %p", cb_function); if (cb_function) { mutex_lock(&gasket_dev->mutex); ret = cb_function(gasket_dev); From 22f2487a1829bfa6b6c88edc7f460f149a480dc6 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:34:55 -0700 Subject: [PATCH 062/209] staging: gasket: remove duplicate call to retrieve device callback gasket_handle_ioctl() calls gasket_get_ioctl_permissions_cb() twice; simplify the code and avoid duplicated work by fetching the callback pointer only once. Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 2e2c9b9..dbe9fde 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -55,14 +55,15 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) { struct gasket_dev *gasket_dev; unsigned long arg = (unsigned long)argp; + gasket_ioctl_permissions_cb_t ioctl_permissions_cb; int retval; gasket_dev = (struct gasket_dev *)filp->private_data; trace_gasket_ioctl_entry(gasket_dev->dev_info.name, cmd); - if (gasket_get_ioctl_permissions_cb(gasket_dev)) { - retval = gasket_get_ioctl_permissions_cb(gasket_dev)( - filp, cmd, argp); + ioctl_permissions_cb = gasket_get_ioctl_permissions_cb(gasket_dev); + if (ioctl_permissions_cb) { + retval = ioctl_permissions_cb(filp, cmd, argp); if (retval < 0) { trace_gasket_ioctl_exit(-EPERM); return retval; From 98e0a3cbf4ff430feaa02b4252cf20fdd5f14996 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:34:56 -0700 Subject: [PATCH 063/209] staging: gasket: gasket_handle_ioctl fix ioctl exit trace param Pass the return value from the device ioctl permissions callback to the tracepoint when the callback returns an error. Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index dbe9fde..1b164ac 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -65,7 +65,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) if (ioctl_permissions_cb) { retval = ioctl_permissions_cb(filp, cmd, argp); if (retval < 0) { - trace_gasket_ioctl_exit(-EPERM); + trace_gasket_ioctl_exit(retval); return retval; } else if (retval == 0) { trace_gasket_ioctl_exit(-EPERM); From 0087db424ac58d4e0e8b18ac0adac4a6357c54a0 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:34:57 -0700 Subject: [PATCH 064/209] staging: gasket: avoid copy to user on error in coherent alloc config gasket_config_coherent_allocator() on error return the error to caller without copying a possibly-update DMA address back to userspace. Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 1b164ac..8cf094b 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -441,8 +441,10 @@ static int gasket_config_coherent_allocator( gasket_dev, ibuf.size, &ibuf.dma_address, ibuf.page_table_index); } + if (ret) + return ret; if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; - return ret; + return 0; } From 3b5a94c0bcd5be3d88ada907e638c338a21a6aff Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:34:58 -0700 Subject: [PATCH 065/209] staging: gasket: print mmap starting address as unsigned long Page alignment error log should print the offending value as an unsigned long, not as a kernel pointer. Reported-by: Guenter Roeck Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 5e269c4..cde9d85 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1606,8 +1606,8 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) if (vma->vm_start & ~PAGE_MASK) { gasket_log_error( - gasket_dev, "Base address not page-aligned: 0x%p\n", - (void *)vma->vm_start); + gasket_dev, "Base address not page-aligned: 0x%lx\n", + vma->vm_start); trace_gasket_mmap_exit(-EINVAL); return -EINVAL; } From b8605d2f862d1bc5672aed5af7fc1f81b3459452 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:34:59 -0700 Subject: [PATCH 066/209] staging: gasket: remove unnecessary NULL checks on calls from VFS Remove unneeded checks for NULL pointers in struct file pointers passed from the VFS layer or the private_data that must have been properly set at file open time. Reported-by: Guenter Roeck Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index cde9d85..06221d1 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1597,11 +1597,6 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) int num_map_regions = 0; enum do_map_region_status map_status; - if (!gasket_dev) { - gasket_nodev_error("Unable to retrieve device data"); - trace_gasket_mmap_exit(-EINVAL); - return -EINVAL; - } driver_desc = gasket_dev->internal_desc->driver_desc; if (vma->vm_start & ~PAGE_MASK) { @@ -1795,17 +1790,7 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) void __user *argp = (void __user *)arg; char path[256]; - if (!filp) - return -ENODEV; - gasket_dev = (struct gasket_dev *)filp->private_data; - if (!gasket_dev) { - gasket_nodev_error( - "Unable to find Gasket structure for file %s", - d_path(&filp->f_path, path, 256)); - return -ENODEV; - } - driver_desc = gasket_dev->internal_desc->driver_desc; if (!driver_desc) { gasket_log_error( From 391b97b101b99dd2c4b261f076350cb67a38c563 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:00 -0700 Subject: [PATCH 067/209] staging: gasket: gasket_get_device drop check for NULL pci_dev The pci_dev field of a struct gasket_dev can never be NULL, there's no need to check for this in gasket_get_device(). Reported-by: Guenter Roeck Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 06221d1..69620eb 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -2036,9 +2036,7 @@ const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev) */ struct device *gasket_get_device(struct gasket_dev *dev) { - if (dev->pci_dev) - return &dev->pci_dev->dev; - return NULL; + return &dev->pci_dev->dev; } /** From 812703123414d84daa2241bf86350c655290e916 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:01 -0700 Subject: [PATCH 068/209] staging: gasket: apex return error on sysfs show of missing attribute Apex sysfs show function return -ENODEV if the attribute is not present, rather than silently failing from the standpoint of the userspace accessor. Reported-by: Guenter Roeck Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index c95071e..3d74611 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -780,14 +780,14 @@ static ssize_t sysfs_show( gasket_dev = gasket_sysfs_get_device_data(device); if (!gasket_dev) { gasket_nodev_error("No Apex device sysfs mapping found"); - return 0; + return -ENODEV; } gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { gasket_nodev_error("No Apex device sysfs attr data found"); gasket_sysfs_put_device_data(device, gasket_dev); - return 0; + return -ENODEV; } type = (enum sysfs_attribute_type)gasket_sysfs_get_attr(device, attr); From f2e730d31245d95070b92df7231c2567acda5046 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:02 -0700 Subject: [PATCH 069/209] staging: gasket: core: convert various logs to debug level Debugging information is improperly logged at non-debug log level in a number of places, and some logs regarding error conditions may be generated too frequently, such that these could cause performance problems and/or obscure other logs. Convert these to debug log level. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 69620eb..c8aefaa 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -205,7 +205,8 @@ static inline int check_and_invoke_callback( { int ret = 0; - gasket_nodev_error("check_and_invoke_callback %p", cb_function); + gasket_log_debug(gasket_dev, "check_and_invoke_callback %p", + cb_function); if (cb_function) { mutex_lock(&gasket_dev->mutex); ret = cb_function(gasket_dev); @@ -227,7 +228,7 @@ static inline int gasket_check_and_invoke_callback_nolock( int ret = 0; if (cb_function) { - gasket_log_info( + gasket_log_debug( gasket_dev, "Invoking device-specific callback."); ret = cb_function(gasket_dev); } @@ -1187,7 +1188,7 @@ static int gasket_release(struct inode *inode, struct file *file) if (file->f_mode & FMODE_WRITE) { ownership->write_open_count--; if (ownership->write_open_count == 0) { - gasket_log_info(gasket_dev, "Device is now free"); + gasket_log_debug(gasket_dev, "Device is now free"); ownership->is_owned = 0; ownership->owner = 0; @@ -1208,7 +1209,7 @@ static int gasket_release(struct inode *inode, struct file *file) } } - gasket_log_info( + gasket_log_debug( gasket_dev, "New open count (owning tgid %u): %d", ownership->owner, ownership->write_open_count); mutex_unlock(&gasket_dev->mutex); @@ -1235,7 +1236,7 @@ static bool gasket_mmap_has_permissions( /* Never allow non-sysadmins to access to a dead device. */ if (gasket_dev->status != GASKET_STATUS_ALIVE) { - gasket_log_info(gasket_dev, "Device is dead."); + gasket_log_debug(gasket_dev, "Device is dead."); return false; } @@ -1243,7 +1244,7 @@ static bool gasket_mmap_has_permissions( requested_permissions = (vma->vm_flags & (VM_WRITE | VM_READ | VM_EXEC)); if (requested_permissions & ~(bar_permissions)) { - gasket_log_info( + gasket_log_debug( gasket_dev, "Attempting to map a region with requested permissions " "0x%x, but region has permissions 0x%x.", @@ -1254,7 +1255,7 @@ static bool gasket_mmap_has_permissions( /* Do not allow a non-owner to write. */ if ((vma->vm_flags & VM_WRITE) && !gasket_owned_by_current_tgid(&gasket_dev->dev_info)) { - gasket_log_info( + gasket_log_debug( gasket_dev, "Attempting to mmap a region for write without owning " "device."); @@ -1746,15 +1747,16 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) status = gasket_check_and_invoke_callback_nolock( gasket_dev, driver_desc->device_status_cb); if (status != GASKET_STATUS_ALIVE) { - gasket_log_info(gasket_dev, "Hardware reported status %d.", - status); + gasket_log_debug(gasket_dev, "Hardware reported status %d.", + status); return status; } status = gasket_interrupt_system_status(gasket_dev); if (status != GASKET_STATUS_ALIVE) { - gasket_log_info(gasket_dev, - "Interrupt system reported status %d.", status); + gasket_log_debug(gasket_dev, + "Interrupt system reported status %d.", + status); return status; } @@ -1762,7 +1764,7 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) status = gasket_page_table_system_status( gasket_dev->page_table[i]); if (status != GASKET_STATUS_ALIVE) { - gasket_log_info( + gasket_log_debug( gasket_dev, "Page table %d reported status %d.", i, status); return status; @@ -1793,7 +1795,7 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) gasket_dev = (struct gasket_dev *)filp->private_data; driver_desc = gasket_dev->internal_desc->driver_desc; if (!driver_desc) { - gasket_log_error( + gasket_log_debug( gasket_dev, "Unable to find device descriptor for file %s", d_path(&filp->f_path, path, 256)); @@ -1841,7 +1843,7 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) /* Perform a device reset of the requested type. */ ret = driver_desc->device_reset_cb(gasket_dev, reset_type); if (ret) { - gasket_log_error( + gasket_log_debug( gasket_dev, "Device reset cb returned %d.", ret); return ret; } @@ -1852,7 +1854,7 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) ret = gasket_interrupt_reinit(gasket_dev); if (ret) { - gasket_log_error( + gasket_log_debug( gasket_dev, "Unable to reinit interrupts: %d.", ret); return ret; } @@ -1860,7 +1862,7 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) /* Get current device health. */ gasket_dev->status = gasket_get_hw_status(gasket_dev); if (gasket_dev->status == GASKET_STATUS_DEAD) { - gasket_log_error(gasket_dev, "Device reported as dead."); + gasket_log_debug(gasket_dev, "Device reported as dead."); return -EINVAL; } @@ -2012,7 +2014,7 @@ static ssize_t gasket_sysfs_data_show( } break; default: - gasket_log_error( + gasket_log_debug( gasket_dev, "Unknown attribute: %s", attr->attr.name); ret = 0; break; @@ -2066,7 +2068,7 @@ int gasket_wait_with_reschedule( msleep(delay_ms); retries++; } - gasket_log_error(gasket_dev, "%s timeout: reg %llx timeout (%llu ms)", + gasket_log_debug(gasket_dev, "%s timeout: reg %llx timeout (%llu ms)", __func__, offset, max_retries * delay_ms); return -ETIMEDOUT; } From 4a6fc715d38842e662b809dce08a277fddea2cf6 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:03 -0700 Subject: [PATCH 070/209] staging: gasket: interrupts: convert various logs to debug level Debugging information is improperly logged at non-debug log level in a number of places, and some logs regarding error conditions may be generated too frequently, such that these could cause performance problems and/or obscure other logs. Convert these to debug log level. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index d096ce7..2b8c26d 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -287,7 +287,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) int ret; if (!gasket_dev->interrupt_data) { - gasket_log_error( + gasket_log_debug( gasket_dev, "Attempted to reinit uninitialized interrupt data."); return -EINVAL; @@ -305,7 +305,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) case PCI_MSI: case PLATFORM_WIRE: default: - gasket_nodev_error( + gasket_nodev_debug( "Cannot handle unsupported interrupt type %d.", gasket_dev->interrupt_data->type); ret = -EINVAL; @@ -351,7 +351,7 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) gasket_dev->interrupt_data; if (!interrupt_data) { - gasket_log_error( + gasket_log_debug( gasket_dev, "Interrupt data is not initialized."); return; } @@ -365,7 +365,7 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } if (interrupt_data->type != PCI_MSIX) { - gasket_nodev_error( + gasket_nodev_debug( "Cannot handle unsupported interrupt type %d.", interrupt_data->type); return; @@ -403,7 +403,7 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) pack_shift = 3 * interrupt_data->pack_width; break; default: - gasket_nodev_error( + gasket_nodev_debug( "Found interrupt description with " "unknown enum %d.", interrupt_data->interrupts[i].packing); @@ -445,7 +445,7 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) case PCI_MSI: case PLATFORM_WIRE: default: - gasket_nodev_error( + gasket_nodev_debug( "Cannot handle unsupported interrupt type %d.", interrupt_data->type); }; @@ -460,18 +460,18 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) { if (!gasket_dev->interrupt_data) { - gasket_nodev_info("Interrupt data is null."); + gasket_nodev_debug("Interrupt data is null."); return GASKET_STATUS_DEAD; } if (!gasket_dev->interrupt_data->msix_configured) { - gasket_nodev_info("Interrupt not initialized."); + gasket_nodev_debug("Interrupt not initialized."); return GASKET_STATUS_LAMED; } if (gasket_dev->interrupt_data->num_configured != gasket_dev->interrupt_data->num_interrupts) { - gasket_nodev_info("Not all interrupts were configured."); + gasket_nodev_debug("Not all interrupts were configured."); return GASKET_STATUS_LAMED; } @@ -516,14 +516,14 @@ static ssize_t interrupt_sysfs_show( gasket_dev = gasket_sysfs_get_device_data(device); if (!gasket_dev) { - gasket_nodev_error( + gasket_nodev_debug( "No sysfs mapping found for device 0x%p", device); return 0; } gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { - gasket_nodev_error( + gasket_nodev_debug( "No sysfs attr data found for device 0x%p", device); gasket_sysfs_put_device_data(device, gasket_dev); return 0; @@ -545,7 +545,7 @@ static ssize_t interrupt_sysfs_show( ret = total_written; break; default: - gasket_log_error( + gasket_log_debug( gasket_dev, "Unknown attribute: %s", attr->attr.name); ret = 0; break; From 93769d3ec3b8ad7883583367debd47afd3f38b07 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:04 -0700 Subject: [PATCH 071/209] staging: gasket: ioctl common: convert various logs to debug level Debugging information is improperly logged at non-debug log level in a number of places, and some logs regarding error conditions may be generated too frequently, such that these could cause performance problems and/or obscure other logs. Convert these to debug log level. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- gasket_ioctl.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index c8aefaa..270815d 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1811,7 +1811,7 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) if (driver_desc->ioctl_handler_cb) return driver_desc->ioctl_handler_cb(filp, cmd, argp); - gasket_log_error( + gasket_log_debug( gasket_dev, "Received unknown ioctl 0x%x", cmd); return -EINVAL; } diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 8cf094b..63e139a 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -73,7 +73,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) } } else if (!gasket_ioctl_check_permissions(filp, cmd)) { trace_gasket_ioctl_exit(-EPERM); - gasket_log_error(gasket_dev, "ioctl cmd=%x noperm.", cmd); + gasket_log_debug(gasket_dev, "ioctl cmd=%x noperm.", cmd); return -EPERM; } @@ -132,7 +132,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) * the arg. */ trace_gasket_ioctl_integer_data(arg); - gasket_log_warn( + gasket_log_debug( gasket_dev, "Unknown ioctl cmd=0x%x not caught by " "gasket_is_supported_ioctl", @@ -329,7 +329,7 @@ static int gasket_partition_page_table( gasket_dev->page_table[ibuf.page_table_index]); if (ibuf.size > max_page_table_size) { - gasket_log_error( + gasket_log_debug( gasket_dev, "Partition request 0x%llx too large, max is 0x%x.", ibuf.size, max_page_table_size); From 31e62a1e10f3c2c8c3aca8a00e790255de88a230 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:05 -0700 Subject: [PATCH 072/209] staging: gasket: page table: convert various logs to debug level Debugging information is improperly logged at non-debug log level in a number of places, and some logs regarding error conditions may be generated too frequently, such that these could cause performance problems and/or obscure other logs. Convert these to debug log level. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 9f81161..f0c4884 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -306,7 +306,7 @@ int gasket_page_table_init( * hardware register that contains the page table size. */ if (total_entries == ULONG_MAX) { - gasket_nodev_error( + gasket_nodev_debug( "Error reading page table size. " "Initializing page table with size 0."); total_entries = 0; @@ -323,7 +323,7 @@ int gasket_page_table_init( *ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL); if (!*ppg_tbl) { - gasket_nodev_error("No memory for page table."); + gasket_nodev_debug("No memory for page table."); return -ENOMEM; } @@ -332,7 +332,7 @@ int gasket_page_table_init( if (bytes != 0) { pg_tbl->entries = vmalloc(bytes); if (!pg_tbl->entries) { - gasket_nodev_error( + gasket_nodev_debug( "No memory for address translation metadata."); kfree(pg_tbl); *ppg_tbl = NULL; @@ -658,7 +658,7 @@ int gasket_page_table_system_status(struct gasket_page_table *page_table) } if (gasket_page_table_num_entries(page_table) == 0) { - gasket_nodev_error("Page table size is 0."); + gasket_nodev_debug("Page table size is 0."); return GASKET_STATUS_LAMED; } @@ -903,7 +903,7 @@ static int gasket_perform_mapping( (unsigned long long)ptes[i].dma_addr); if (ptes[i].dma_addr == -1) { - gasket_nodev_error( + gasket_nodev_debug( "%s i %d" " -> fail to map page %llx " "[pfn %p ohys %p]\n", @@ -1612,7 +1612,7 @@ int gasket_set_user_virt( */ pg_tbl = gasket_dev->page_table[0]; if (!pg_tbl) { - gasket_nodev_error( + gasket_nodev_debug( "%s: invalid page table index", __func__); return 0; } From f6ac13d5f32edad4ac77e1e40c177dc2bfe9640a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:06 -0700 Subject: [PATCH 073/209] staging: gasket: page table: remove unnecessary logs Some error logs in page table handling code could only be hit in cases of programming errors not expected in the current code base, and aren't likely to be useful on their own. Remove these. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index f0c4884..4f2ff77 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -605,10 +605,8 @@ EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad); /* See gasket_page_table.h for description. */ uint gasket_page_table_max_size(struct gasket_page_table *page_table) { - if (!page_table) { - gasket_nodev_error("Passed a null page table."); + if (!page_table) return 0; - } return page_table->config.total_entries; } EXPORT_SYMBOL(gasket_page_table_max_size); @@ -616,11 +614,8 @@ EXPORT_SYMBOL(gasket_page_table_max_size); /* See gasket_page_table.h for description. */ uint gasket_page_table_num_entries(struct gasket_page_table *pg_tbl) { - if (!pg_tbl) { - gasket_nodev_error("Passed a null page table."); + if (!pg_tbl) return 0; - } - return pg_tbl->num_simple_entries + pg_tbl->num_extended_entries; } EXPORT_SYMBOL(gasket_page_table_num_entries); @@ -628,11 +623,8 @@ EXPORT_SYMBOL(gasket_page_table_num_entries); /* See gasket_page_table.h for description. */ uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl) { - if (!pg_tbl) { - gasket_nodev_error("Passed a null page table."); + if (!pg_tbl) return 0; - } - return pg_tbl->num_simple_entries; } EXPORT_SYMBOL(gasket_page_table_num_simple_entries); @@ -640,11 +632,8 @@ EXPORT_SYMBOL(gasket_page_table_num_simple_entries); /* See gasket_page_table.h for description. */ uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl) { - if (!pg_tbl) { - gasket_nodev_error("Passed a null page table."); + if (!pg_tbl) return 0; - } - return pg_tbl->num_active_pages; } EXPORT_SYMBOL(gasket_page_table_num_active_pages); @@ -652,10 +641,8 @@ EXPORT_SYMBOL(gasket_page_table_num_active_pages); /* See gasket_page_table.h */ int gasket_page_table_system_status(struct gasket_page_table *page_table) { - if (!page_table) { - gasket_nodev_error("Passed a null page table."); + if (!page_table) return GASKET_STATUS_LAMED; - } if (gasket_page_table_num_entries(page_table) == 0) { gasket_nodev_debug("Page table size is 0."); From b04dc30709b818c44c20092791e123d7674571ca Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 21 Jul 2018 06:35:07 -0700 Subject: [PATCH 074/209] staging: gasket: apex: convert various logs to debug level Debugging information is improperly logged at non-debug log level in a number of places, and some logs regarding error conditions may be generated too frequently, such that these could cause performance problems and/or obscure other logs. Convert these to debug log level. Signed-off-by: Zhongze Hu Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 3d74611..b4d3696 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -422,7 +422,7 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); - gasket_log_info( + gasket_log_debug( gasket_dev, "%s 0x%p hib_error 0x%llx scalar_error " "0x%llx.", @@ -591,7 +591,7 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); val1 = gasket_dev_read_32( gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); - gasket_log_error( + gasket_log_debug( gasket_dev, "Disallow HW clock gating 0x%x -> 0x%x", val0, val1); } else { @@ -604,7 +604,7 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); val1 = gasket_dev_read_32( gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); - gasket_log_error( + gasket_log_debug( gasket_dev, "Allow HW clock gating 0x%x -> 0x%x", val0, val1); } @@ -672,7 +672,7 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, if (copy_from_user(&ibuf, argp, sizeof(ibuf))) return -EFAULT; - gasket_log_error(gasket_dev, "%s %llu", __func__, ibuf.enable); + gasket_log_debug(gasket_dev, "%s %llu", __func__, ibuf.enable); if (ibuf.enable) { /* Quiesce AXI, gate GCB clock. */ @@ -808,7 +808,7 @@ static ssize_t sysfs_show( gasket_dev->page_table[0])); break; default: - gasket_log_error( + gasket_log_debug( gasket_dev, "Unknown attribute: %s", attr->attr.name); ret = 0; break; From eb6c6b5f4854795839b8ba7b601e0d0d093ee858 Mon Sep 17 00:00:00 2001 From: Ivan Bornyakov Date: Mon, 23 Jul 2018 21:30:25 +0300 Subject: [PATCH 075/209] staging: gasket: use vzalloc instead of vmalloc/memset Use vzalloc instead of vmalloc followed by memset with 0. Signed-off-by: Ivan Bornyakov Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 4f2ff77..55ab593 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -330,7 +330,7 @@ int gasket_page_table_init( pg_tbl = *ppg_tbl; bytes = total_entries * sizeof(struct gasket_page_table_entry); if (bytes != 0) { - pg_tbl->entries = vmalloc(bytes); + pg_tbl->entries = vzalloc(bytes); if (!pg_tbl->entries) { gasket_nodev_debug( "No memory for address translation metadata."); @@ -338,7 +338,6 @@ int gasket_page_table_init( *ppg_tbl = NULL; return -ENOMEM; } - memset(pg_tbl->entries, 0, bytes); } mutex_init(&pg_tbl->mutex); @@ -1054,13 +1053,12 @@ static int gasket_alloc_extended_subtable( subtable_bytes = sizeof(struct gasket_page_table_entry) * GASKET_PAGES_PER_SUBTABLE; - pte->sublevel = vmalloc(subtable_bytes); + pte->sublevel = vzalloc(subtable_bytes); if (!pte->sublevel) { free_page(page_addr); memset(pte, 0, sizeof(struct gasket_page_table_entry)); return -ENOMEM; } - memset(pte->sublevel, 0, subtable_bytes); /* Map the page into DMA space. */ if (pg_tbl->dma_ops) { From c7fc63a2e94fce7ac2e70db620e9e6e4b50de074 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:28 -0700 Subject: [PATCH 076/209] staging: gasket: save struct device for a gasket device Save the struct device pointer to a gasket device in gasket's metadata, to facilitate use of standard logging calls and in anticipation of non-PCI gasket devices in the future. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 5 +++-- gasket_core.h | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 270815d..e47d963 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -450,6 +450,7 @@ static int gasket_alloc_dev( gasket_dev->internal_desc = internal_desc; gasket_dev->dev_idx = dev_idx; snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", kobj_name); + gasket_dev->dev = parent; /* gasket_bar_data is uninitialized. */ gasket_dev->num_page_tables = driver_desc->num_page_tables; /* max_page_table_size and *page table are uninit'ed */ @@ -935,7 +936,7 @@ static int gasket_enable_dev( &gasket_dev->bar_data[ driver_desc->page_table_bar_index], &driver_desc->page_table_configs[tbl_idx], - &gasket_dev->pci_dev->dev, gasket_dev->pci_dev, true); + gasket_dev->dev, gasket_dev->pci_dev, true); if (ret) { gasket_log_error( gasket_dev, @@ -2038,7 +2039,7 @@ const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev) */ struct device *gasket_get_device(struct gasket_dev *dev) { - return &dev->pci_dev->dev; + return dev->dev; } /** diff --git a/gasket_core.h b/gasket_core.h index bf4ed37..8bd431a 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -263,6 +263,9 @@ struct gasket_dev { /* Pointer to the internal driver description for this device. */ struct gasket_internal_desc *internal_desc; + /* Device info */ + struct device *dev; + /* PCI subsystem metadata. */ struct pci_dev *pci_dev; From 8b9b0eb845418aa29f5204a41718d5a27342495c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:29 -0700 Subject: [PATCH 077/209] staging: gasket: core: convert to standard logging Use standard logging functions, drop use of gasket log functions. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 295 +++++++++++++++++++++++--------------------------- 1 file changed, 134 insertions(+), 161 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index e47d963..6beee37 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -10,15 +10,16 @@ #include "gasket_interrupt.h" #include "gasket_ioctl.h" -#include "gasket_logging.h" #include "gasket_page_table.h" #include "gasket_sysfs.h" #include #include +#include #include #include #include +#include #ifdef GASKET_KERNEL_TRACE_SUPPORT #define CREATE_TRACE_POINTS @@ -205,8 +206,8 @@ static inline int check_and_invoke_callback( { int ret = 0; - gasket_log_debug(gasket_dev, "check_and_invoke_callback %p", - cb_function); + dev_dbg(gasket_dev->dev, "check_and_invoke_callback %p\n", + cb_function); if (cb_function) { mutex_lock(&gasket_dev->mutex); ret = cb_function(gasket_dev); @@ -228,8 +229,8 @@ static inline int gasket_check_and_invoke_callback_nolock( int ret = 0; if (cb_function) { - gasket_log_debug( - gasket_dev, "Invoking device-specific callback."); + dev_dbg(gasket_dev->dev, + "Invoking device-specific callback.\n"); ret = cb_function(gasket_dev); } return ret; @@ -250,7 +251,7 @@ static int __init gasket_init(void) { int i; - gasket_nodev_info("Performing one-time init of the Gasket framework."); + pr_info("Performing one-time init of the Gasket framework.\n"); /* Check for duplicates and find a free slot. */ mutex_lock(&g_mutex); for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { @@ -267,7 +268,7 @@ static int __init gasket_init(void) static void __exit gasket_exit(void) { /* No deinit/dealloc needed at present. */ - gasket_nodev_info("Removing Gasket framework module."); + pr_info("Removing Gasket framework module.\n"); } /* See gasket_core.h for description. */ @@ -277,15 +278,14 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) int desc_idx = -1; struct gasket_internal_desc *internal; - gasket_nodev_info("Initializing Gasket framework device"); + pr_info("Initializing Gasket framework device\n"); /* Check for duplicates and find a free slot. */ mutex_lock(&g_mutex); for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { if (g_descs[i].driver_desc == driver_desc) { - gasket_nodev_error( - "%s driver already loaded/registered", - driver_desc->name); + pr_err("%s driver already loaded/registered\n", + driver_desc->name); mutex_unlock(&g_mutex); return -EBUSY; } @@ -301,17 +301,17 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) } mutex_unlock(&g_mutex); - gasket_nodev_info("Loaded %s driver, framework version %s", - driver_desc->name, GASKET_FRAMEWORK_VERSION); + pr_info("Loaded %s driver, framework version %s\n", + driver_desc->name, GASKET_FRAMEWORK_VERSION); if (desc_idx == -1) { - gasket_nodev_error("Too many Gasket drivers loaded: %d\n", - GASKET_FRAMEWORK_DESC_MAX); + pr_err("Too many Gasket drivers loaded: %d\n", + GASKET_FRAMEWORK_DESC_MAX); return -EBUSY; } /* Internal structure setup. */ - gasket_nodev_info("Performing initial internal structure setup."); + pr_debug("Performing initial internal structure setup.\n"); internal = &g_descs[desc_idx]; mutex_init(&internal->mutex); memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX); @@ -324,8 +324,8 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) class_create(driver_desc->module, driver_desc->name); if (IS_ERR(internal->class)) { - gasket_nodev_error("Cannot register %s class [ret=%ld]", - driver_desc->name, PTR_ERR(internal->class)); + pr_err("Cannot register %s class [ret=%ld]\n", + driver_desc->name, PTR_ERR(internal->class)); ret = PTR_ERR(internal->class); goto unregister_gasket_driver; } @@ -334,25 +334,24 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) * Not using pci_register_driver() (without underscores), as it * depends on KBUILD_MODNAME, and this is a shared file. */ - gasket_nodev_info("Registering PCI driver."); + pr_debug("Registering PCI driver.\n"); ret = __pci_register_driver( &internal->pci, driver_desc->module, driver_desc->name); if (ret) { - gasket_nodev_error( - "cannot register pci driver [ret=%d]", ret); + pr_err("cannot register pci driver [ret=%d]\n", ret); goto fail1; } - gasket_nodev_info("Registering char driver."); + pr_debug("Registering char driver.\n"); ret = register_chrdev_region( MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, driver_desc->name); if (ret) { - gasket_nodev_error("cannot register char driver [ret=%d]", ret); + pr_err("cannot register char driver [ret=%d]\n", ret); goto fail2; } - gasket_nodev_info("Driver registered successfully."); + pr_info("Driver registered successfully.\n"); return 0; fail2: @@ -386,10 +385,9 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) mutex_unlock(&g_mutex); if (!internal_desc) { - gasket_nodev_error( - "request to unregister unknown desc: %s, %d:%d", - driver_desc->name, driver_desc->major, - driver_desc->minor); + pr_err("request to unregister unknown desc: %s, %d:%d\n", + driver_desc->name, driver_desc->major, + driver_desc->minor); return; } @@ -405,7 +403,7 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) g_descs[desc_idx].driver_desc = NULL; mutex_unlock(&g_mutex); - gasket_nodev_info("removed %s driver", driver_desc->name); + pr_info("removed %s driver\n", driver_desc->name); } EXPORT_SYMBOL(gasket_unregister_device); @@ -430,7 +428,7 @@ static int gasket_alloc_dev( struct gasket_dev *gasket_dev; struct gasket_cdev_info *dev_info; - gasket_nodev_info("Allocating a Gasket device %s.", kobj_name); + pr_debug("Allocating a Gasket device %s.\n", kobj_name); *pdev = NULL; @@ -440,7 +438,7 @@ static int gasket_alloc_dev( gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL); if (!gasket_dev) { - gasket_nodev_error("no memory for device"); + pr_err("no memory for device\n"); return -ENOMEM; } internal_desc->devs[dev_idx] = gasket_dev; @@ -466,7 +464,7 @@ static int gasket_alloc_dev( dev_info->device = device_create(internal_desc->class, parent, dev_info->devt, gasket_dev, dev_info->name); - gasket_nodev_info("Gasket device allocated: %p.", dev_info->device); + dev_dbg(dev_info->device, "Gasket device allocated.\n"); /* cdev has not yet been added; cdev_added is 0 */ dev_info->gasket_dev_ptr = gasket_dev; @@ -509,7 +507,7 @@ static int gasket_find_dev_slot( for (i = 0; i < GASKET_DEV_MAX; i++) { if (internal_desc->devs[i] && strcmp(internal_desc->devs[i]->kobj_name, kobj_name) == 0) { - gasket_nodev_error("Duplicate device %s", kobj_name); + pr_err("Duplicate device %s\n", kobj_name); mutex_unlock(&internal_desc->mutex); return -EBUSY; } @@ -522,8 +520,7 @@ static int gasket_find_dev_slot( } if (i == GASKET_DEV_MAX) { - gasket_nodev_info( - "Too many registered devices; max %d", GASKET_DEV_MAX); + pr_err("Too many registered devices; max %d\n", GASKET_DEV_MAX); mutex_unlock(&internal_desc->mutex); return -EBUSY; } @@ -552,13 +549,13 @@ static int gasket_pci_probe( const struct gasket_driver_desc *driver_desc; struct device *parent; - gasket_nodev_info("Add Gasket device %s", kobj_name); + pr_info("Add Gasket device %s\n", kobj_name); mutex_lock(&g_mutex); internal_desc = lookup_internal_desc(pci_dev); mutex_unlock(&g_mutex); if (!internal_desc) { - gasket_nodev_info("PCI probe called for unknown driver type"); + pr_err("PCI probe called for unknown driver type\n"); return -ENODEV; } @@ -569,9 +566,9 @@ static int gasket_pci_probe( if (ret) return ret; if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { - gasket_nodev_error("Cannot create %s device %s [ret = %ld]", - driver_desc->name, gasket_dev->dev_info.name, - PTR_ERR(gasket_dev->dev_info.device)); + pr_err("Cannot create %s device %s [ret = %ld]\n", + driver_desc->name, gasket_dev->dev_info.name, + PTR_ERR(gasket_dev->dev_info.device)); ret = -ENODEV; goto fail1; } @@ -583,7 +580,7 @@ static int gasket_pci_probe( ret = check_and_invoke_callback(gasket_dev, driver_desc->add_dev_cb); if (ret) { - gasket_log_error(gasket_dev, "Error in add device cb: %d", ret); + dev_err(gasket_dev->dev, "Error in add device cb: %d\n", ret); goto fail2; } @@ -599,8 +596,8 @@ static int gasket_pci_probe( ret = sysfs_create_link(&gasket_dev->dev_info.device->kobj, &pci_dev->dev.kobj, dev_name(&pci_dev->dev)); if (ret) { - gasket_log_error( - gasket_dev, "Cannot create sysfs pci link: %d", ret); + dev_err(gasket_dev->dev, + "Cannot create sysfs pci link: %d\n", ret); goto fail3; } ret = gasket_sysfs_create_entries( @@ -611,14 +608,13 @@ static int gasket_pci_probe( ret = check_and_invoke_callback( gasket_dev, driver_desc->sysfs_setup_cb); if (ret) { - gasket_log_error( - gasket_dev, "Error in sysfs setup cb: %d", ret); + dev_err(gasket_dev->dev, "Error in sysfs setup cb: %d\n", ret); goto fail5; } ret = gasket_enable_dev(internal_desc, gasket_dev); if (ret) { - gasket_nodev_error("cannot setup %s device", driver_desc->name); + pr_err("cannot setup %s device\n", driver_desc->name); gasket_disable_dev(gasket_dev); goto fail5; } @@ -687,8 +683,7 @@ static void gasket_pci_remove(struct pci_dev *pci_dev) if (!gasket_dev) return; - gasket_nodev_info( - "remove %s device %s", internal_desc->driver_desc->name, + pr_info("remove %s device %s\n", internal_desc->driver_desc->name, gasket_dev->kobj_name); gasket_disable_dev(gasket_dev); @@ -721,7 +716,7 @@ static int gasket_setup_pci( gasket_dev->pci_dev = pci_dev; ret = pci_enable_device(pci_dev); if (ret) { - gasket_log_error(gasket_dev, "cannot enable PCI device"); + dev_err(gasket_dev->dev, "cannot enable PCI device\n"); return ret; } @@ -787,17 +782,16 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) gasket_dev->bar_data[bar_num].phys_base = (ulong)pci_resource_start(gasket_dev->pci_dev, bar_num); if (!gasket_dev->bar_data[bar_num].phys_base) { - gasket_log_error(gasket_dev, "Cannot get BAR%u base address", - bar_num); + dev_err(gasket_dev->dev, "Cannot get BAR%u base address\n", + bar_num); return -EINVAL; } gasket_dev->bar_data[bar_num].length_bytes = (ulong)pci_resource_len(gasket_dev->pci_dev, bar_num); if (gasket_dev->bar_data[bar_num].length_bytes < desc_bytes) { - gasket_log_error( - gasket_dev, - "PCI BAR %u space is too small: %lu; expected >= %lu", + dev_err(gasket_dev->dev, + "PCI BAR %u space is too small: %lu; expected >= %lu\n", bar_num, gasket_dev->bar_data[bar_num].length_bytes, desc_bytes); return -ENOMEM; @@ -806,9 +800,8 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) if (!request_mem_region(gasket_dev->bar_data[bar_num].phys_base, gasket_dev->bar_data[bar_num].length_bytes, gasket_dev->dev_info.name)) { - gasket_log_error( - gasket_dev, - "Cannot get BAR %d memory region %p", + dev_err(gasket_dev->dev, + "Cannot get BAR %d memory region %p\n", bar_num, &gasket_dev->pci_dev->resource[bar_num]); return -EINVAL; } @@ -817,9 +810,8 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) ioremap_nocache(gasket_dev->bar_data[bar_num].phys_base, gasket_dev->bar_data[bar_num].length_bytes); if (!gasket_dev->bar_data[bar_num].virt_base) { - gasket_log_error( - gasket_dev, - "Cannot remap BAR %d memory region %p", + dev_err(gasket_dev->dev, + "Cannot remap BAR %d memory region %p\n", bar_num, &gasket_dev->pci_dev->resource[bar_num]); ret = -ENOMEM; goto fail; @@ -862,8 +854,8 @@ static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num) base = pci_resource_start(dev->pci_dev, bar_num); if (!base) { - gasket_log_error( - dev, "cannot get PCI BAR%u base address", bar_num); + dev_err(dev->dev, "cannot get PCI BAR%u base address\n", + bar_num); return; } @@ -887,9 +879,8 @@ static int gasket_add_cdev( dev_info->cdev.owner = owner; ret = cdev_add(&dev_info->cdev, dev_info->devt, 1); if (ret) { - gasket_log_error( - dev_info->gasket_dev_ptr, - "cannot add char device [ret=%d]", ret); + dev_err(dev_info->gasket_dev_ptr->dev, + "cannot add char device [ret=%d]\n", ret); return ret; } dev_info->cdev_added = 1; @@ -921,16 +912,15 @@ static int gasket_enable_dev( driver_desc->interrupt_bar_index, driver_desc->wire_interrupt_offsets); if (ret) { - gasket_log_error(gasket_dev, - "Critical failure to allocate interrupts: %d", - ret); + dev_err(gasket_dev->dev, + "Critical failure to allocate interrupts: %d\n", ret); gasket_interrupt_cleanup(gasket_dev); return ret; } for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { - gasket_log_debug( - gasket_dev, "Initializing page table %d.", tbl_idx); + dev_dbg(gasket_dev->dev, "Initializing page table %d.\n", + tbl_idx); ret = gasket_page_table_init( &gasket_dev->page_table[tbl_idx], &gasket_dev->bar_data[ @@ -938,9 +928,8 @@ static int gasket_enable_dev( &driver_desc->page_table_configs[tbl_idx], gasket_dev->dev, gasket_dev->pci_dev, true); if (ret) { - gasket_log_error( - gasket_dev, - "Couldn't init page table %d: %d", + dev_err(gasket_dev->dev, + "Couldn't init page table %d: %d\n", tbl_idx, ret); return ret; } @@ -958,23 +947,23 @@ static int gasket_enable_dev( ret = check_and_invoke_callback( gasket_dev, driver_desc->hardware_revision_cb); if (ret < 0) { - gasket_log_error( - gasket_dev, "Error getting hardware revision: %d", ret); + dev_err(gasket_dev->dev, + "Error getting hardware revision: %d\n", ret); return ret; } gasket_dev->hardware_revision = ret; ret = check_and_invoke_callback(gasket_dev, driver_desc->enable_dev_cb); if (ret) { - gasket_log_error( - gasket_dev, "Error in enable device cb: %d", ret); + dev_err(gasket_dev->dev, "Error in enable device cb: %d\n", + ret); return ret; } /* device_status_cb returns a device status, not an error code. */ gasket_dev->status = gasket_get_hw_status(gasket_dev); if (gasket_dev->status == GASKET_STATUS_DEAD) - gasket_log_error(gasket_dev, "Device reported as unhealthy."); + dev_err(gasket_dev->dev, "Device reported as unhealthy.\n"); ret = gasket_add_cdev( &gasket_dev->dev_info, &gasket_file_ops, driver_desc->module); @@ -1094,31 +1083,29 @@ static int gasket_open(struct inode *inode, struct file *filp) filp->private_data = gasket_dev; inode->i_size = 0; - gasket_log_debug( - gasket_dev, + dev_dbg(gasket_dev->dev, "Attempting to open with tgid %u (%s) (f_mode: 0%03o, " - "fmode_write: %d is_root: %u)", + "fmode_write: %d is_root: %u)\n", current->tgid, task_name, filp->f_mode, (filp->f_mode & FMODE_WRITE), is_root); /* Always allow non-writing accesses. */ if (!(filp->f_mode & FMODE_WRITE)) { - gasket_log_debug(gasket_dev, "Allowing read-only opening."); + dev_dbg(gasket_dev->dev, "Allowing read-only opening.\n"); return 0; } mutex_lock(&gasket_dev->mutex); - gasket_log_debug( - gasket_dev, "Current owner open count (owning tgid %u): %d.", + dev_dbg(gasket_dev->dev, + "Current owner open count (owning tgid %u): %d.\n", ownership->owner, ownership->write_open_count); /* Opening a node owned by another TGID is an error (unless root) */ if (ownership->is_owned && ownership->owner != current->tgid && !is_root) { - gasket_log_error( - gasket_dev, - "Process %u is opening a node held by %u.", + dev_err(gasket_dev->dev, + "Process %u is opening a node held by %u.\n", current->tgid, ownership->owner); mutex_unlock(&gasket_dev->mutex); return -EPERM; @@ -1129,21 +1116,21 @@ static int gasket_open(struct inode *inode, struct file *filp) ret = gasket_check_and_invoke_callback_nolock( gasket_dev, driver_desc->device_open_cb); if (ret) { - gasket_log_error( - gasket_dev, "Error in device open cb: %d", ret); + dev_err(gasket_dev->dev, + "Error in device open cb: %d\n", ret); mutex_unlock(&gasket_dev->mutex); return ret; } ownership->is_owned = 1; ownership->owner = current->tgid; - gasket_log_debug(gasket_dev, "Device owner is now tgid %u", - ownership->owner); + dev_dbg(gasket_dev->dev, "Device owner is now tgid %u\n", + ownership->owner); } ownership->write_open_count++; - gasket_log_debug(gasket_dev, "New open count (owning tgid %u): %d", - ownership->owner, ownership->write_open_count); + dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n", + ownership->owner, ownership->write_open_count); mutex_unlock(&gasket_dev->mutex); return 0; @@ -1177,19 +1164,18 @@ static int gasket_release(struct inode *inode, struct file *file) get_task_comm(task_name, current); mutex_lock(&gasket_dev->mutex); - gasket_log_debug( - gasket_dev, + dev_dbg(gasket_dev->dev, "Releasing device node. Call origin: tgid %u (%s) " - "(f_mode: 0%03o, fmode_write: %d, is_root: %u)", + "(f_mode: 0%03o, fmode_write: %d, is_root: %u)\n", current->tgid, task_name, file->f_mode, (file->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN)); - gasket_log_debug(gasket_dev, "Current open count (owning tgid %u): %d", - ownership->owner, ownership->write_open_count); + dev_dbg(gasket_dev->dev, "Current open count (owning tgid %u): %d\n", + ownership->owner, ownership->write_open_count); if (file->f_mode & FMODE_WRITE) { ownership->write_open_count--; if (ownership->write_open_count == 0) { - gasket_log_debug(gasket_dev, "Device is now free"); + dev_dbg(gasket_dev->dev, "Device is now free\n"); ownership->is_owned = 0; ownership->owner = 0; @@ -1210,8 +1196,7 @@ static int gasket_release(struct inode *inode, struct file *file) } } - gasket_log_debug( - gasket_dev, "New open count (owning tgid %u): %d", + dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n", ownership->owner, ownership->write_open_count); mutex_unlock(&gasket_dev->mutex); return 0; @@ -1237,7 +1222,7 @@ static bool gasket_mmap_has_permissions( /* Never allow non-sysadmins to access to a dead device. */ if (gasket_dev->status != GASKET_STATUS_ALIVE) { - gasket_log_debug(gasket_dev, "Device is dead."); + dev_dbg(gasket_dev->dev, "Device is dead.\n"); return false; } @@ -1245,10 +1230,9 @@ static bool gasket_mmap_has_permissions( requested_permissions = (vma->vm_flags & (VM_WRITE | VM_READ | VM_EXEC)); if (requested_permissions & ~(bar_permissions)) { - gasket_log_debug( - gasket_dev, + dev_dbg(gasket_dev->dev, "Attempting to map a region with requested permissions " - "0x%x, but region has permissions 0x%x.", + "0x%x, but region has permissions 0x%x.\n", requested_permissions, bar_permissions); return false; } @@ -1256,10 +1240,9 @@ static bool gasket_mmap_has_permissions( /* Do not allow a non-owner to write. */ if ((vma->vm_flags & VM_WRITE) && !gasket_owned_by_current_tgid(&gasket_dev->dev_info)) { - gasket_log_debug( - gasket_dev, + dev_dbg(gasket_dev->dev, "Attempting to mmap a region for write without owning " - "device."); + "device.\n"); return false; } @@ -1472,8 +1455,8 @@ static enum do_map_region_status do_map_region( (phys_base + mapped_bytes) >> PAGE_SHIFT, chunk_size, vma->vm_page_prot); if (ret) { - gasket_log_error( - gasket_dev, "Error remapping PFN range."); + dev_err(gasket_dev->dev, + "Error remapping PFN range.\n"); goto fail; } mapped_bytes += chunk_size; @@ -1485,9 +1468,8 @@ fail: /* Unmap the partial chunk we mapped. */ mappable_region->length_bytes = mapped_bytes; if (gasket_mm_unmap_region(gasket_dev, vma, mappable_region)) - gasket_log_error( - gasket_dev, - "Error unmapping partial region 0x%lx (0x%lx bytes)", + dev_err(gasket_dev->dev, + "Error unmapping partial region 0x%lx (0x%lx bytes)\n", (ulong)virt_offset, (ulong)mapped_bytes); @@ -1512,9 +1494,8 @@ static int gasket_mm_vma_bar_offset( driver_desc->legacy_mmap_address_offset; bar_index = gasket_get_bar_index(gasket_dev, raw_offset); if (bar_index < 0) { - gasket_log_error( - gasket_dev, - "Unable to find matching bar for address 0x%lx", + dev_err(gasket_dev->dev, + "Unable to find matching bar for address 0x%lx\n", raw_offset); trace_gasket_mmap_exit(bar_index); return bar_index; @@ -1547,7 +1528,7 @@ static int gasket_mmap_coherent( permissions = driver_desc->coherent_buffer_description.permissions; if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) { - gasket_log_error(gasket_dev, "Permission checking failed."); + dev_err(gasket_dev->dev, "Permission checking failed.\n"); trace_gasket_mmap_exit(-EPERM); return -EPERM; } @@ -1559,8 +1540,8 @@ static int gasket_mmap_coherent( (gasket_dev->coherent_buffer.phys_base) >> PAGE_SHIFT, requested_length, vma->vm_page_prot); if (ret) { - gasket_log_error( - gasket_dev, "Error remapping PFN range err=%d.", ret); + dev_err(gasket_dev->dev, "Error remapping PFN range err=%d.\n", + ret); trace_gasket_mmap_exit(ret); return ret; } @@ -1602,8 +1583,8 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) driver_desc = gasket_dev->internal_desc->driver_desc; if (vma->vm_start & ~PAGE_MASK) { - gasket_log_error( - gasket_dev, "Base address not page-aligned: 0x%lx\n", + dev_err(gasket_dev->dev, + "Base address not page-aligned: 0x%lx\n", vma->vm_start); trace_gasket_mmap_exit(-EINVAL); return -EINVAL; @@ -1623,18 +1604,16 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) bar_index = gasket_get_bar_index(gasket_dev, raw_offset); is_coherent_region = gasket_is_coherent_region(driver_desc, raw_offset); if (bar_index < 0 && !is_coherent_region) { - gasket_log_error( - gasket_dev, - "Unable to find matching bar for address 0x%lx", + dev_err(gasket_dev->dev, + "Unable to find matching bar for address 0x%lx\n", raw_offset); trace_gasket_mmap_exit(bar_index); return bar_index; } if (bar_index > 0 && is_coherent_region) { - gasket_log_error( - gasket_dev, + dev_err(gasket_dev->dev, "double matching bar and coherent buffers for address " - "0x%lx", + "0x%lx\n", raw_offset); trace_gasket_mmap_exit(bar_index); return -EINVAL; @@ -1654,7 +1633,7 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) bar_desc = &driver_desc->bar_descriptions[bar_index]; permissions = bar_desc->permissions; if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) { - gasket_log_error(gasket_dev, "Permission checking failed."); + dev_err(gasket_dev->dev, "Permission checking failed.\n"); trace_gasket_mmap_exit(-EPERM); return -EPERM; } @@ -1667,8 +1646,8 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) } else { if (!gasket_mmap_has_permissions(gasket_dev, vma, bar_desc->permissions)) { - gasket_log_error( - gasket_dev, "Permission checking failed."); + dev_err(gasket_dev->dev, + "Permission checking failed.\n"); trace_gasket_mmap_exit(-EPERM); return -EPERM; } @@ -1684,7 +1663,7 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) } if (!map_regions || num_map_regions == 0) { - gasket_log_error(gasket_dev, "No mappable regions returned!"); + dev_err(gasket_dev->dev, "No mappable regions returned!\n"); return -EINVAL; } @@ -1707,9 +1686,8 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) /* If we could not map any memory, the request was invalid. */ if (!has_mapped_anything) { - gasket_log_error( - gasket_dev, - "Map request did not contain a valid region."); + dev_err(gasket_dev->dev, + "Map request did not contain a valid region.\n"); trace_gasket_mmap_exit(-EINVAL); return -EINVAL; } @@ -1723,8 +1701,8 @@ fail: for (i = 0; i < num_map_regions; i++) if (gasket_mm_unmap_region(gasket_dev, vma, &bar_desc->mappable_regions[i])) - gasket_log_error( - gasket_dev, "Error unmapping range %d.", i); + dev_err(gasket_dev->dev, "Error unmapping range %d.\n", + i); kfree(map_regions); return ret; @@ -1748,16 +1726,15 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) status = gasket_check_and_invoke_callback_nolock( gasket_dev, driver_desc->device_status_cb); if (status != GASKET_STATUS_ALIVE) { - gasket_log_debug(gasket_dev, "Hardware reported status %d.", - status); + dev_dbg(gasket_dev->dev, "Hardware reported status %d.\n", + status); return status; } status = gasket_interrupt_system_status(gasket_dev); if (status != GASKET_STATUS_ALIVE) { - gasket_log_debug(gasket_dev, - "Interrupt system reported status %d.", - status); + dev_dbg(gasket_dev->dev, + "Interrupt system reported status %d.\n", status); return status; } @@ -1765,8 +1742,8 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) status = gasket_page_table_system_status( gasket_dev->page_table[i]); if (status != GASKET_STATUS_ALIVE) { - gasket_log_debug( - gasket_dev, "Page table %d reported status %d.", + dev_dbg(gasket_dev->dev, + "Page table %d reported status %d.\n", i, status); return status; } @@ -1796,9 +1773,8 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) gasket_dev = (struct gasket_dev *)filp->private_data; driver_desc = gasket_dev->internal_desc->driver_desc; if (!driver_desc) { - gasket_log_debug( - gasket_dev, - "Unable to find device descriptor for file %s", + dev_dbg(gasket_dev->dev, + "Unable to find device descriptor for file %s\n", d_path(&filp->f_path, path, 256)); return -ENODEV; } @@ -1812,8 +1788,7 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) if (driver_desc->ioctl_handler_cb) return driver_desc->ioctl_handler_cb(filp, cmd, argp); - gasket_log_debug( - gasket_dev, "Received unknown ioctl 0x%x", cmd); + dev_dbg(gasket_dev->dev, "Received unknown ioctl 0x%x\n", cmd); return -EINVAL; } @@ -1844,8 +1819,8 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) /* Perform a device reset of the requested type. */ ret = driver_desc->device_reset_cb(gasket_dev, reset_type); if (ret) { - gasket_log_debug( - gasket_dev, "Device reset cb returned %d.", ret); + dev_dbg(gasket_dev->dev, "Device reset cb returned %d.\n", + ret); return ret; } @@ -1855,15 +1830,15 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) ret = gasket_interrupt_reinit(gasket_dev); if (ret) { - gasket_log_debug( - gasket_dev, "Unable to reinit interrupts: %d.", ret); + dev_dbg(gasket_dev->dev, "Unable to reinit interrupts: %d.\n", + ret); return ret; } /* Get current device health. */ gasket_dev->status = gasket_get_hw_status(gasket_dev); if (gasket_dev->status == GASKET_STATUS_DEAD) { - gasket_log_debug(gasket_dev, "Device reported as dead."); + dev_dbg(gasket_dev->dev, "Device reported as dead.\n"); return -EINVAL; } @@ -1919,15 +1894,13 @@ static ssize_t gasket_sysfs_data_show( gasket_dev = gasket_sysfs_get_device_data(device); if (!gasket_dev) { - gasket_nodev_error( - "No sysfs mapping found for device 0x%p", device); + dev_err(device, "No sysfs mapping found for device\n"); return 0; } gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { - gasket_nodev_error( - "No sysfs attr found for device 0x%p", device); + dev_err(device, "No sysfs attr found for device\n"); gasket_sysfs_put_device_data(device, gasket_dev); return 0; } @@ -2015,8 +1988,8 @@ static ssize_t gasket_sysfs_data_show( } break; default: - gasket_log_debug( - gasket_dev, "Unknown attribute: %s", attr->attr.name); + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); ret = 0; break; } @@ -2069,8 +2042,8 @@ int gasket_wait_with_reschedule( msleep(delay_ms); retries++; } - gasket_log_debug(gasket_dev, "%s timeout: reg %llx timeout (%llu ms)", - __func__, offset, max_retries * delay_ms); + dev_dbg(gasket_dev->dev, "%s timeout: reg %llx timeout (%llu ms)\n", + __func__, offset, max_retries * delay_ms); return -ETIMEDOUT; } EXPORT_SYMBOL(gasket_wait_with_reschedule); From ed5e450ff4733cd989f9c8bf7f8fc0b7465be599 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:30 -0700 Subject: [PATCH 078/209] staging: gasket: interrupt: convert to standard logging Convert gasket logging calls to standard functions. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 67 ++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 2b8c26d..3be8e24 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -5,9 +5,10 @@ #include "gasket_constants.h" #include "gasket_core.h" -#include "gasket_logging.h" #include "gasket_sysfs.h" +#include #include +#include #include #ifdef GASKET_KERNEL_TRACE_SUPPORT #define CREATE_TRACE_POINTS @@ -165,8 +166,8 @@ int gasket_interrupt_init( case PCI_MSI: case PLATFORM_WIRE: default: - gasket_nodev_error( - "Cannot handle unsupported interrupt type %d.", + dev_err(gasket_dev->dev, + "Cannot handle unsupported interrupt type %d\n", interrupt_data->type); ret = -EINVAL; }; @@ -175,8 +176,8 @@ int gasket_interrupt_init( /* Failing to setup interrupts will cause the device to report * GASKET_STATUS_LAMED. But it is not fatal. */ - gasket_log_warn( - gasket_dev, "Couldn't initialize interrupts: %d", ret); + dev_warn(gasket_dev->dev, + "Couldn't initialize interrupts: %d\n", ret); return 0; } @@ -216,7 +217,7 @@ static int gasket_interrupt_msix_init( interrupt_data); if (ret) { - gasket_nodev_error( + dev_err(&interrupt_data->pci_dev->dev, "Cannot get IRQ for interrupt %d, vector %d; " "%d\n", i, interrupt_data->msix_entries[i].vector, ret); @@ -287,9 +288,8 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) int ret; if (!gasket_dev->interrupt_data) { - gasket_log_debug( - gasket_dev, - "Attempted to reinit uninitialized interrupt data."); + dev_dbg(gasket_dev->dev, + "Attempted to reinit uninitialized interrupt data\n"); return -EINVAL; } @@ -305,8 +305,8 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) case PCI_MSI: case PLATFORM_WIRE: default: - gasket_nodev_debug( - "Cannot handle unsupported interrupt type %d.", + dev_dbg(gasket_dev->dev, + "Cannot handle unsupported interrupt type %d\n", gasket_dev->interrupt_data->type); ret = -EINVAL; }; @@ -315,7 +315,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) /* Failing to setup MSIx will cause the device * to report GASKET_STATUS_LAMED, but is not fatal. */ - gasket_log_warn(gasket_dev, "Couldn't init msix: %d", ret); + dev_warn(gasket_dev->dev, "Couldn't init msix: %d\n", ret); return 0; } @@ -327,7 +327,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) /* See gasket_interrupt.h for description. */ int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev) { - gasket_log_debug(gasket_dev, "Clearing interrupt counts."); + dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n"); memset(gasket_dev->interrupt_data->interrupt_counts, 0, gasket_dev->interrupt_data->num_interrupts * sizeof(*gasket_dev->interrupt_data->interrupt_counts)); @@ -351,12 +351,11 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) gasket_dev->interrupt_data; if (!interrupt_data) { - gasket_log_debug( - gasket_dev, "Interrupt data is not initialized."); + dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n"); return; } - gasket_log_debug(gasket_dev, "Running interrupt setup."); + dev_dbg(gasket_dev->dev, "Running interrupt setup\n"); if (interrupt_data->type == PLATFORM_WIRE || interrupt_data->type == PCI_MSI) { @@ -365,8 +364,8 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } if (interrupt_data->type != PCI_MSIX) { - gasket_nodev_debug( - "Cannot handle unsupported interrupt type %d.", + dev_dbg(gasket_dev->dev, + "Cannot handle unsupported interrupt type %d\n", interrupt_data->type); return; } @@ -379,10 +378,9 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) * the register directly. If not, we need to deal with a read- * modify-write and shift based on the packing index. */ - gasket_log_debug( - gasket_dev, + dev_dbg(gasket_dev->dev, "Setting up interrupt index %d with index 0x%llx and " - "packing %d", + "packing %d\n", interrupt_data->interrupts[i].index, interrupt_data->interrupts[i].reg, interrupt_data->interrupts[i].packing); @@ -403,9 +401,9 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) pack_shift = 3 * interrupt_data->pack_width; break; default: - gasket_nodev_debug( + dev_dbg(gasket_dev->dev, "Found interrupt description with " - "unknown enum %d.", + "unknown enum %d\n", interrupt_data->interrupts[i].packing); return; } @@ -445,8 +443,8 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) case PCI_MSI: case PLATFORM_WIRE: default: - gasket_nodev_debug( - "Cannot handle unsupported interrupt type %d.", + dev_dbg(gasket_dev->dev, + "Cannot handle unsupported interrupt type %d\n", interrupt_data->type); }; @@ -460,18 +458,19 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) { if (!gasket_dev->interrupt_data) { - gasket_nodev_debug("Interrupt data is null."); + dev_dbg(gasket_dev->dev, "Interrupt data is null\n"); return GASKET_STATUS_DEAD; } if (!gasket_dev->interrupt_data->msix_configured) { - gasket_nodev_debug("Interrupt not initialized."); + dev_dbg(gasket_dev->dev, "Interrupt not initialized\n"); return GASKET_STATUS_LAMED; } if (gasket_dev->interrupt_data->num_configured != gasket_dev->interrupt_data->num_interrupts) { - gasket_nodev_debug("Not all interrupts were configured."); + dev_dbg(gasket_dev->dev, + "Not all interrupts were configured\n"); return GASKET_STATUS_LAMED; } @@ -516,15 +515,13 @@ static ssize_t interrupt_sysfs_show( gasket_dev = gasket_sysfs_get_device_data(device); if (!gasket_dev) { - gasket_nodev_debug( - "No sysfs mapping found for device 0x%p", device); + dev_dbg(device, "No sysfs mapping found for device\n"); return 0; } gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { - gasket_nodev_debug( - "No sysfs attr data found for device 0x%p", device); + dev_dbg(device, "No sysfs attr data found for device\n"); gasket_sysfs_put_device_data(device, gasket_dev); return 0; } @@ -545,8 +542,8 @@ static ssize_t interrupt_sysfs_show( ret = total_written; break; default: - gasket_log_debug( - gasket_dev, "Unknown attribute: %s", attr->attr.name); + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); ret = 0; break; } @@ -574,7 +571,7 @@ static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) } } if (interrupt == -1) { - gasket_nodev_error("Received unknown irq %d", irq); + pr_err("Received unknown irq %d\n", irq); return IRQ_HANDLED; } trace_gasket_interrupt_event(interrupt_data->name, interrupt); From 8b9f03116bfc8b1e4b06b18dfe7af22ce50d457b Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:31 -0700 Subject: [PATCH 079/209] staging: gasket: ioctl: convert to standard logging Replace gasket logging calls with standard logging calls. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 63e139a..78a132a 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -5,9 +5,9 @@ #include "gasket_constants.h" #include "gasket_core.h" #include "gasket_interrupt.h" -#include "gasket_logging.h" #include "gasket_page_table.h" #include +#include #include #include @@ -73,7 +73,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) } } else if (!gasket_ioctl_check_permissions(filp, cmd)) { trace_gasket_ioctl_exit(-EPERM); - gasket_log_debug(gasket_dev, "ioctl cmd=%x noperm.", cmd); + dev_dbg(gasket_dev->dev, "ioctl cmd=%x noperm\n", cmd); return -EPERM; } @@ -132,10 +132,9 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) * the arg. */ trace_gasket_ioctl_integer_data(arg); - gasket_log_debug( - gasket_dev, + dev_dbg(gasket_dev->dev, "Unknown ioctl cmd=0x%x not caught by " - "gasket_is_supported_ioctl", + "gasket_is_supported_ioctl\n", cmd); retval = -EINVAL; break; @@ -186,12 +185,9 @@ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; alive = (gasket_dev->status == GASKET_STATUS_ALIVE); - if (!alive) { - gasket_nodev_error( - "%s alive %d status %d.", - __func__, - alive, gasket_dev->status); - } + if (!alive) + dev_dbg(gasket_dev->dev, "%s alive %d status %d\n", + __func__, alive, gasket_dev->status); read = !!(filp->f_mode & FMODE_READ); write = !!(filp->f_mode & FMODE_WRITE); @@ -329,9 +325,8 @@ static int gasket_partition_page_table( gasket_dev->page_table[ibuf.page_table_index]); if (ibuf.size > max_page_table_size) { - gasket_log_debug( - gasket_dev, - "Partition request 0x%llx too large, max is 0x%x.", + dev_dbg(gasket_dev->dev, + "Partition request 0x%llx too large, max is 0x%x\n", ibuf.size, max_page_table_size); return -EINVAL; } From 213eabfa36fcc7dc3d646be591555d678460c44d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:32 -0700 Subject: [PATCH 080/209] staging: gasket: page table: convert to standard logging Replace gasket logging calls with standard logging calls. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 131 ++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 77 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 55ab593..8ea8ea1 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -33,6 +33,7 @@ */ #include "gasket_page_table.h" +#include #include #include #include @@ -43,7 +44,6 @@ #include "gasket_constants.h" #include "gasket_core.h" -#include "gasket_logging.h" /* Constants & utility macros */ /* The number of pages that can be mapped into each second-level page table. */ @@ -79,11 +79,6 @@ */ #define GASKET_EXTENDED_LVL1_SHIFT 12 -/* Page-table specific error logging. */ -#define gasket_pg_tbl_error(pg_tbl, format, arg...) \ - gasket_dev_log(err, (pg_tbl)->device, (struct pci_dev *)NULL, format, \ - ##arg) - /* Type declarations */ /* Valid states for a struct gasket_page_table_entry. */ enum pte_status { @@ -306,24 +301,23 @@ int gasket_page_table_init( * hardware register that contains the page table size. */ if (total_entries == ULONG_MAX) { - gasket_nodev_debug( - "Error reading page table size. " - "Initializing page table with size 0."); + dev_dbg(device, "Error reading page table size. " + "Initializing page table with size 0\n"); total_entries = 0; } - gasket_nodev_debug( - "Attempting to initialize page table of size 0x%lx.", + dev_dbg(device, + "Attempting to initialize page table of size 0x%lx\n", total_entries); - gasket_nodev_debug( - "Table has base reg 0x%x, extended offset reg 0x%x.", + dev_dbg(device, + "Table has base reg 0x%x, extended offset reg 0x%x\n", page_table_config->base_reg, page_table_config->extended_reg); *ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL); if (!*ppg_tbl) { - gasket_nodev_debug("No memory for page table."); + dev_dbg(device, "No memory for page table\n"); return -ENOMEM; } @@ -332,8 +326,8 @@ int gasket_page_table_init( if (bytes != 0) { pg_tbl->entries = vzalloc(bytes); if (!pg_tbl->entries) { - gasket_nodev_debug( - "No memory for address translation metadata."); + dev_dbg(device, + "No memory for address translation metadata\n"); kfree(pg_tbl); *ppg_tbl = NULL; return -ENOMEM; @@ -361,7 +355,7 @@ int gasket_page_table_init( pg_tbl->pci_dev = pci_dev; pg_tbl->dma_ops = has_dma_ops; - gasket_nodev_debug("Page table initialized successfully."); + dev_dbg(device, "Page table initialized successfully\n"); return 0; } @@ -398,7 +392,7 @@ int gasket_page_table_partition( for (i = start; i < pg_tbl->config.total_entries; i++) { if (pg_tbl->entries[i].status != PTE_FREE) { - gasket_pg_tbl_error(pg_tbl, "entry %d is not free", i); + dev_err(pg_tbl->device, "entry %d is not free\n", i); mutex_unlock(&pg_tbl->mutex); return -EBUSY; } @@ -443,11 +437,9 @@ int gasket_page_table_map( mutex_unlock(&pg_tbl->mutex); - gasket_nodev_debug( - "%s done: ha %llx daddr %llx num %d, " - "ret %d\n", - __func__, - (unsigned long long)host_addr, + dev_dbg(pg_tbl->device, + "%s done: ha %llx daddr %llx num %d, ret %d\n", + __func__, (unsigned long long)host_addr, (unsigned long long)dev_addr, num_pages, ret); return ret; } @@ -562,9 +554,8 @@ bool gasket_page_table_are_addrs_bad( ulong bytes) { if (host_addr & (PAGE_SIZE - 1)) { - gasket_pg_tbl_error( - pg_tbl, - "host mapping address 0x%lx must be page aligned", + dev_err(pg_tbl->device, + "host mapping address 0x%lx must be page aligned\n", host_addr); return true; } @@ -580,16 +571,14 @@ bool gasket_page_table_is_dev_addr_bad( uint num_pages = bytes / PAGE_SIZE; if (bytes & (PAGE_SIZE - 1)) { - gasket_pg_tbl_error( - pg_tbl, - "mapping size 0x%lX must be page aligned", bytes); + dev_err(pg_tbl->device, + "mapping size 0x%lX must be page aligned\n", bytes); return true; } if (num_pages == 0) { - gasket_pg_tbl_error( - pg_tbl, - "requested mapping is less than one page: %lu / %lu", + dev_err(pg_tbl->device, + "requested mapping is less than one page: %lu / %lu\n", bytes, PAGE_SIZE); return true; } @@ -644,7 +633,7 @@ int gasket_page_table_system_status(struct gasket_page_table *page_table) return GASKET_STATUS_LAMED; if (gasket_page_table_num_entries(page_table) == 0) { - gasket_nodev_debug("Page table size is 0."); + dev_dbg(page_table->device, "Page table size is 0\n"); return GASKET_STATUS_LAMED; } @@ -682,9 +671,8 @@ static int gasket_map_simple_pages( ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); if (ret) { - gasket_pg_tbl_error( - pg_tbl, - "page table slots %u (@ 0x%lx) to %u are not available", + dev_err(pg_tbl->device, + "page table slots %u (@ 0x%lx) to %u are not available\n", slot_idx, dev_addr, slot_idx + num_pages - 1); return ret; } @@ -695,7 +683,7 @@ static int gasket_map_simple_pages( if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); - gasket_pg_tbl_error(pg_tbl, "gasket_perform_mapping %d.", ret); + dev_err(pg_tbl->device, "gasket_perform_mapping %d\n", ret); } return ret; } @@ -732,10 +720,9 @@ static int gasket_map_extended_pages( ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages); if (ret) { dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; - gasket_pg_tbl_error( - pg_tbl, + dev_err(pg_tbl->device, "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are " - "not available", + "not available\n", gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), dev_addr, gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), @@ -843,7 +830,7 @@ static int gasket_perform_mapping( for (i = 0; i < num_pages; i++) { page_addr = host_addr + i * PAGE_SIZE; offset = page_addr & (PAGE_SIZE - 1); - gasket_nodev_debug("%s i %d\n", __func__, i); + dev_dbg(pg_tbl->device, "%s i %d\n", __func__, i); if (is_coherent(pg_tbl, host_addr)) { u64 off = (u64)host_addr - @@ -857,10 +844,9 @@ static int gasket_perform_mapping( page_addr - offset, 1, 1, &page); if (ret <= 0) { - gasket_pg_tbl_error( - pg_tbl, + dev_err(pg_tbl->device, "get user pages failed for addr=0x%lx, " - "offset=0x%lx [ret=%d]", + "offset=0x%lx [ret=%d]\n", page_addr, offset, ret); return ret ? ret : -ENOMEM; } @@ -880,21 +866,17 @@ static int gasket_perform_mapping( DMA_BIDIRECTIONAL); } - gasket_nodev_debug( - "%s dev %p " - "i %d pte %p pfn %p -> mapped %llx\n", - __func__, - pg_tbl->device, i, &ptes[i], + dev_dbg(pg_tbl->device, + "%s i %d pte %p pfn %p -> mapped %llx\n", + __func__, i, &ptes[i], (void *)page_to_pfn(page), (unsigned long long)ptes[i].dma_addr); if (ptes[i].dma_addr == -1) { - gasket_nodev_debug( - "%s i %d" - " -> fail to map page %llx " + dev_dbg(pg_tbl->device, + "%s i %d -> fail to map page %llx " "[pfn %p ohys %p]\n", - __func__, - i, + __func__, i, (unsigned long long)ptes[i].dma_addr, (void *)page_to_pfn(page), (void *)page_to_phys(page)); @@ -996,9 +978,8 @@ static int gasket_alloc_extended_entries( if (pte->status == PTE_FREE) { ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); if (ret) { - gasket_pg_tbl_error( - pg_tbl, - "no memory for extended addr subtable"); + dev_err(pg_tbl->device, + "no memory for extended addr subtable\n"); return ret; } } else { @@ -1309,23 +1290,21 @@ static bool gasket_is_simple_dev_addr_bad( if (gasket_components_to_dev_address( pg_tbl, 1, page_index, page_offset) != dev_addr) { - gasket_pg_tbl_error( - pg_tbl, "address is invalid, 0x%lX", dev_addr); + dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", + dev_addr); return true; } if (page_index >= pg_tbl->num_simple_entries) { - gasket_pg_tbl_error( - pg_tbl, - "starting slot at %lu is too large, max is < %u", + dev_err(pg_tbl->device, + "starting slot at %lu is too large, max is < %u\n", page_index, pg_tbl->num_simple_entries); return true; } if (page_index + num_pages > pg_tbl->num_simple_entries) { - gasket_pg_tbl_error( - pg_tbl, - "ending slot at %lu is too large, max is <= %u", + dev_err(pg_tbl->device, + "ending slot at %lu is too large, max is <= %u\n", page_index + num_pages, pg_tbl->num_simple_entries); return true; } @@ -1354,8 +1333,8 @@ static bool gasket_is_extended_dev_addr_bad( /* check if the device address is out of bound */ addr = dev_addr & ~((pg_tbl)->extended_flag); if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - gasket_pg_tbl_error(pg_tbl, "device address out of bound, 0x%p", - (void *)dev_addr); + dev_err(pg_tbl->device, "device address out of bound, 0x%p\n", + (void *)dev_addr); return true; } @@ -1372,23 +1351,21 @@ static bool gasket_is_extended_dev_addr_bad( if (gasket_components_to_dev_address( pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { - gasket_pg_tbl_error( - pg_tbl, "address is invalid, 0x%p", (void *)dev_addr); + dev_err(pg_tbl->device, "address is invalid, 0x%p\n", + (void *)dev_addr); return true; } if (page_lvl0_idx >= pg_tbl->num_extended_entries) { - gasket_pg_tbl_error( - pg_tbl, + dev_err(pg_tbl->device, "starting level 0 slot at %lu is too large, max is < " - "%u", page_lvl0_idx, pg_tbl->num_extended_entries); + "%u\n", page_lvl0_idx, pg_tbl->num_extended_entries); return true; } if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) { - gasket_pg_tbl_error( - pg_tbl, - "ending level 0 slot at %lu is too large, max is <= %u", + dev_err(pg_tbl->device, + "ending level 0 slot at %lu is too large, max is <= %u\n", page_lvl0_idx + num_lvl0_pages, pg_tbl->num_extended_entries); return true; @@ -1597,8 +1574,8 @@ int gasket_set_user_virt( */ pg_tbl = gasket_dev->page_table[0]; if (!pg_tbl) { - gasket_nodev_debug( - "%s: invalid page table index", __func__); + dev_dbg(gasket_dev->dev, "%s: invalid page table index\n", + __func__); return 0; } for (j = 0; j < num_pages; j++) { From 158ef348af1543941336c93b6403464b048eb861 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:33 -0700 Subject: [PATCH 081/209] staging: gasket: sysfs: convert to standard logging Drop gasket logging calls in favor of standard logging. Change-Id: Icd23c6c43b671cb8380b36bf26500bc835eec2c7 Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 73 ++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 197bf53..666c876 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -3,7 +3,9 @@ #include "gasket_sysfs.h" #include "gasket_core.h" -#include "gasket_logging.h" + +#include +#include /* * Pair of kernel device and user-specified pointer. Used in lookups in sysfs @@ -66,7 +68,7 @@ static struct gasket_sysfs_mapping *get_mapping(struct device *device) int i; if (!device) { - gasket_nodev_error("Received NULL device!"); + pr_debug("%s: Received NULL device\n", __func__); return NULL; } @@ -80,7 +82,8 @@ static struct gasket_sysfs_mapping *get_mapping(struct device *device) mutex_unlock(&dev_mappings[i].mutex); } - gasket_nodev_info("Mapping to device %s not found.", device->kobj.name); + dev_dbg(device, "%s: Mapping to device %s not found\n", + __func__, device->kobj.name); return NULL; } @@ -103,16 +106,15 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) struct device *device; if (!mapping) { - gasket_nodev_info("Mapping should not be NULL."); + pr_debug("%s: Mapping should not be NULL\n", __func__); return; } mutex_lock(&mapping->mutex); if (atomic_read(&mapping->refcount.refcount) == 0) - gasket_nodev_error("Refcount is already 0!"); + dev_err(mapping->device, "Refcount is already 0\n"); if (kref_put(&mapping->refcount, release_entry)) { - gasket_nodev_info("Removing Gasket sysfs mapping, device %s", - mapping->device->kobj.name); + dev_dbg(mapping->device, "Removing Gasket sysfs mapping\n"); /* * We can't remove the sysfs nodes in the kref callback, since * device_remove_file() blocks until the node is free. @@ -182,16 +184,13 @@ int gasket_sysfs_create_mapping( static DEFINE_MUTEX(function_mutex); mutex_lock(&function_mutex); - - gasket_nodev_info( - "Creating sysfs entries for device pointer 0x%p.", device); + dev_dbg(device, "Creating sysfs entries for device\n"); /* Check that the device we're adding hasn't already been added. */ mapping = get_mapping(device); if (mapping) { - gasket_nodev_error( - "Attempting to re-initialize sysfs mapping for device " - "0x%p.", device); + dev_err(device, + "Attempting to re-initialize sysfs mapping for device\n"); put_mapping(mapping); mutex_unlock(&function_mutex); return -EBUSY; @@ -207,13 +206,13 @@ int gasket_sysfs_create_mapping( } if (map_idx == GASKET_SYSFS_NUM_MAPPINGS) { - gasket_nodev_error("All mappings have been exhausted!"); + dev_err(device, "All mappings have been exhausted\n"); mutex_unlock(&function_mutex); return -ENOMEM; } - gasket_nodev_info( - "Creating sysfs mapping for device %s.", device->kobj.name); + dev_dbg(device, "Creating sysfs mapping for device %s\n", + device->kobj.name); mapping = &dev_mappings[map_idx]; kref_init(&mapping->refcount); @@ -224,7 +223,7 @@ int gasket_sysfs_create_mapping( GFP_KERNEL); mapping->attribute_count = 0; if (!mapping->attributes) { - gasket_nodev_error("Unable to allocate sysfs attribute array."); + dev_dbg(device, "Unable to allocate sysfs attribute array\n"); mapping->device = NULL; mapping->gasket_dev = NULL; mutex_unlock(&mapping->mutex); @@ -247,10 +246,9 @@ int gasket_sysfs_create_entries( struct gasket_sysfs_mapping *mapping = get_mapping(device); if (!mapping) { - gasket_nodev_error( - "Creating entries for device 0x%p without first " - "initializing mapping.", - device); + dev_dbg(device, + "Creating entries for device without first " + "initializing mapping\n"); return -EINVAL; } @@ -258,9 +256,9 @@ int gasket_sysfs_create_entries( for (i = 0; strcmp(attrs[i].attr.attr.name, GASKET_ARRAY_END_MARKER); i++) { if (mapping->attribute_count == GASKET_SYSFS_MAX_NODES) { - gasket_nodev_error( + dev_err(device, "Maximum number of sysfs nodes reached for " - "device."); + "device\n"); mutex_unlock(&mapping->mutex); put_mapping(mapping); return -ENOMEM; @@ -268,7 +266,7 @@ int gasket_sysfs_create_entries( ret = device_create_file(device, &attrs[i].attr); if (ret) { - gasket_nodev_error("Unable to create device entries"); + dev_dbg(device, "Unable to create device entries\n"); mutex_unlock(&mapping->mutex); put_mapping(mapping); return ret; @@ -289,10 +287,9 @@ void gasket_sysfs_remove_mapping(struct device *device) struct gasket_sysfs_mapping *mapping = get_mapping(device); if (!mapping) { - gasket_nodev_error( + dev_err(device, "Attempted to remove non-existent sysfs mapping to " - "device 0x%p", - device); + "device\n"); return; } @@ -304,7 +301,7 @@ struct gasket_dev *gasket_sysfs_get_device_data(struct device *device) struct gasket_sysfs_mapping *mapping = get_mapping(device); if (!mapping) { - gasket_nodev_error("device %p not registered.", device); + dev_err(device, "device not registered\n"); return NULL; } @@ -342,8 +339,8 @@ struct gasket_sysfs_attribute *gasket_sysfs_get_attr( return &attrs[i]; } - gasket_nodev_error("Unable to find match for device_attribute %s", - attr->attr.name); + dev_err(device, "Unable to find match for device_attribute %s\n", + attr->attr.name); return NULL; } EXPORT_SYMBOL(gasket_sysfs_get_attr); @@ -368,8 +365,8 @@ void gasket_sysfs_put_attr( } } - gasket_nodev_error( - "Unable to put unknown attribute: %s", attr->attr.attr.name); + dev_err(device, "Unable to put unknown attribute: %s\n", + attr->attr.attr.name); } EXPORT_SYMBOL(gasket_sysfs_put_attr); @@ -383,26 +380,26 @@ ssize_t gasket_sysfs_register_store( struct gasket_sysfs_attribute *gasket_attr; if (count < 3 || buf[0] != '0' || buf[1] != 'x') { - gasket_nodev_error( - "sysfs register write format: \"0x\"."); + dev_err(device, + "sysfs register write format: \"0x\"\n"); return -EINVAL; } if (kstrtoul(buf, 16, &parsed_value) != 0) { - gasket_nodev_error( - "Unable to parse input as 64-bit hex value: %s.", buf); + dev_err(device, + "Unable to parse input as 64-bit hex value: %s\n", buf); return -EINVAL; } mapping = get_mapping(device); if (!mapping) { - gasket_nodev_info("Device driver may have been removed."); + dev_err(device, "Device driver may have been removed\n"); return 0; } gasket_dev = mapping->gasket_dev; if (!gasket_dev) { - gasket_nodev_info("Device driver may have been removed."); + dev_err(device, "Device driver may have been removed\n"); return 0; } From 57a55b53ab1cf5f10daf17855c1f44c198a46665 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:34 -0700 Subject: [PATCH 082/209] staging: gasket: apex: convert to standard logging Drop gasket logging calls in favor of standard logging. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 61 ++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index b4d3696..7a2646a 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -7,11 +7,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include @@ -19,7 +21,6 @@ #include "gasket_core.h" #include "gasket_interrupt.h" -#include "gasket_logging.h" #include "gasket_page_table.h" #include "gasket_sysfs.h" @@ -364,11 +365,9 @@ static int apex_add_dev_cb(struct gasket_dev *gasket_dev) if (retries == APEX_RESET_RETRY) { if (!page_table_ready) - gasket_log_error( - gasket_dev, "Page table init timed out."); + dev_err(gasket_dev->dev, "Page table init timed out\n"); if (!msix_table_ready) - gasket_log_error( - gasket_dev, "MSI-X table init timed out."); + dev_err(gasket_dev->dev, "MSI-X table init timed out\n"); return -ETIMEDOUT; } @@ -422,12 +421,9 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); - gasket_log_debug( - gasket_dev, - "%s 0x%p hib_error 0x%llx scalar_error " - "0x%llx.", - __func__, - gasket_dev, hib_error, scalar_error); + dev_dbg(gasket_dev->dev, + "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n", + __func__, gasket_dev, hib_error, scalar_error); if (allow_power_save) ret = apex_enter_reset(gasket_dev, APEX_CHIP_REINIT_RESET); @@ -454,7 +450,7 @@ static int apex_reset(struct gasket_dev *gasket_dev, uint type) /* We are not in reset - toggle the reset bit so as to force * re-init of custom block */ - gasket_log_debug(gasket_dev, "%s: toggle reset.", __func__); + dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__); ret = apex_enter_reset(gasket_dev, type); if (ret) @@ -491,9 +487,9 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1, APEX_RESET_DELAY, APEX_RESET_RETRY)) { - gasket_log_error(gasket_dev, - "DMAs did not quiesce within timeout (%d ms)", - APEX_RESET_RETRY * APEX_RESET_DELAY); + dev_err(gasket_dev->dev, + "DMAs did not quiesce within timeout (%d ms)\n", + APEX_RESET_RETRY * APEX_RESET_DELAY); return -ETIMEDOUT; } @@ -513,9 +509,8 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 1 << 6, 1 << 6, APEX_RESET_DELAY, APEX_RESET_RETRY)) { - gasket_log_error( - gasket_dev, - "RAM did not shut down within timeout (%d ms)", + dev_err(gasket_dev->dev, + "RAM did not shut down within timeout (%d ms)\n", APEX_RESET_RETRY * APEX_RESET_DELAY); return -ETIMEDOUT; } @@ -562,9 +557,8 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 1 << 6, 0, APEX_RESET_DELAY, APEX_RESET_RETRY)) { - gasket_log_error( - gasket_dev, - "RAM did not enable within timeout (%d ms)", + dev_err(gasket_dev->dev, + "RAM did not enable within timeout (%d ms)\n", APEX_RESET_RETRY * APEX_RESET_DELAY); return -ETIMEDOUT; } @@ -574,9 +568,8 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) APEX_BAR2_REG_SCU_3, SCU3_CUR_RST_GCB_BIT_MASK, 0, APEX_RESET_DELAY, APEX_RESET_RETRY)) { - gasket_log_error( - gasket_dev, - "GCB did not leave reset within timeout (%d ms)", + dev_err(gasket_dev->dev, + "GCB did not leave reset within timeout (%d ms)\n", APEX_RESET_RETRY * APEX_RESET_DELAY); return -ETIMEDOUT; } @@ -591,9 +584,8 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); val1 = gasket_dev_read_32( gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); - gasket_log_debug( - gasket_dev, "Disallow HW clock gating 0x%x -> 0x%x", - val0, val1); + dev_dbg(gasket_dev->dev, + "Disallow HW clock gating 0x%x -> 0x%x\n", val0, val1); } else { val0 = gasket_dev_read_32( gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); @@ -604,9 +596,8 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); val1 = gasket_dev_read_32( gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); - gasket_log_debug( - gasket_dev, "Allow HW clock gating 0x%x -> 0x%x", val0, - val1); + dev_dbg(gasket_dev->dev, "Allow HW clock gating 0x%x -> 0x%x\n", + val0, val1); } return 0; @@ -672,7 +663,7 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, if (copy_from_user(&ibuf, argp, sizeof(ibuf))) return -EFAULT; - gasket_log_debug(gasket_dev, "%s %llu", __func__, ibuf.enable); + dev_dbg(gasket_dev->dev, "%s %llu\n", __func__, ibuf.enable); if (ibuf.enable) { /* Quiesce AXI, gate GCB clock. */ @@ -779,13 +770,13 @@ static ssize_t sysfs_show( gasket_dev = gasket_sysfs_get_device_data(device); if (!gasket_dev) { - gasket_nodev_error("No Apex device sysfs mapping found"); + dev_err(device, "No Apex device sysfs mapping found\n"); return -ENODEV; } gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { - gasket_nodev_error("No Apex device sysfs attr data found"); + dev_err(device, "No Apex device sysfs attr data found\n"); gasket_sysfs_put_device_data(device, gasket_dev); return -ENODEV; } @@ -808,8 +799,8 @@ static ssize_t sysfs_show( gasket_dev->page_table[0])); break; default: - gasket_log_debug( - gasket_dev, "Unknown attribute: %s", attr->attr.name); + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); ret = 0; break; } From 047c7f4fc734189df03843ea61f0cc202c37c0f9 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:35 -0700 Subject: [PATCH 083/209] staging: gasket: remove gasket logging header Gasket logging functions no longer used. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_logging.h | 64 ------------------------------------------------ 1 file changed, 64 deletions(-) delete mode 100644 gasket_logging.h diff --git a/gasket_logging.h b/gasket_logging.h deleted file mode 100644 index 54bbe51..0000000 --- a/gasket_logging.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Common logging utilities for the Gasket driver framework. - * - * Copyright (C) 2018 Google, Inc. - */ - -#include -#include - -#ifndef _GASKET_LOGGING_H_ -#define _GASKET_LOGGING_H_ - -/* Base macro; other logging can/should be built on top of this. */ -#define gasket_dev_log(level, device, pci_dev, format, arg...) \ - if (false) { \ - if (pci_dev) { \ - dev_##level(&(pci_dev)->dev, "%s: " format "\n", \ - __func__, ##arg); \ - } else { \ - gasket_nodev_log(level, format, ##arg); \ - } \ - } - -/* "No-device" logging macros. */ -#define gasket_nodev_log(level, format, arg...) \ - if (false) pr_##level("gasket: %s: " format "\n", __func__, ##arg) - -#define gasket_nodev_debug(format, arg...) \ - if (false) gasket_nodev_log(debug, format, ##arg) - -#define gasket_nodev_info(format, arg...) \ - if (false) gasket_nodev_log(info, format, ##arg) - -#define gasket_nodev_warn(format, arg...) \ - if (false) gasket_nodev_log(warn, format, ##arg) - -#define gasket_nodev_error(format, arg...) \ - if (false) gasket_nodev_log(err, format, ##arg) - -/* gasket_dev logging macros */ -#define gasket_log_info(gasket_dev, format, arg...) \ - if (false) gasket_dev_log(info, (gasket_dev)->dev_info.device, \ - (gasket_dev)->pci_dev, format, ##arg) - -#define gasket_log_warn(gasket_dev, format, arg...) \ - if (false) gasket_dev_log(warn, (gasket_dev)->dev_info.device, \ - (gasket_dev)->pci_dev, format, ##arg) - -#define gasket_log_error(gasket_dev, format, arg...) \ - if (false) gasket_dev_log(err, (gasket_dev)->dev_info.device, \ - (gasket_dev)->pci_dev, format, ##arg) - -#define gasket_log_debug(gasket_dev, format, arg...) \ - if (false){ \ - if ((gasket_dev)->pci_dev) { \ - dev_dbg(&((gasket_dev)->pci_dev)->dev, "%s: " format \ - "\n", __func__, ##arg); \ - } else { \ - gasket_nodev_log(debug, format, ##arg); \ - } \ - } - -#endif From 371a3a39a3921be706bcf2ab13610e6f77eae3dd Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:36 -0700 Subject: [PATCH 084/209] staging: gasket: TODO: remove entry for convert to standard logging Gasket/apex drivers now use standard logging, remove TODO entry for this. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index d3c44ca..fb71997 100644 --- a/TODO +++ b/TODO @@ -4,7 +4,6 @@ staging directory. - Document sysfs files with Documentation/ABI/ entries. - Use misc interface instead of major number for driver version description. - Add descriptions of module_param's -- Remove gasket-specific logging functions. - apex_get_status() should actually check status. - Static functions don't need kernel doc formatting, can be simplified. - Fix multi-line alignment formatting to look like: From 5420904bf3986ba86bb4aac5e49ed47607545265 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 26 Jul 2018 20:07:37 -0700 Subject: [PATCH 085/209] staging: gasket: don't print device addresses as kernel pointers Print device addresses as unsigned long, not as kernel pointers. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 8ea8ea1..32f1c1e 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1333,8 +1333,8 @@ static bool gasket_is_extended_dev_addr_bad( /* check if the device address is out of bound */ addr = dev_addr & ~((pg_tbl)->extended_flag); if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - dev_err(pg_tbl->device, "device address out of bound, 0x%p\n", - (void *)dev_addr); + dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n", + dev_addr); return true; } @@ -1351,8 +1351,8 @@ static bool gasket_is_extended_dev_addr_bad( if (gasket_components_to_dev_address( pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid, 0x%p\n", - (void *)dev_addr); + dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", + dev_addr); return true; } From 84a04811154c41aaa3a8ef1d7aee247963adeb55 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Fri, 27 Jul 2018 22:21:56 -0700 Subject: [PATCH 086/209] staging: gasket: sysfs: remove check for refcount already zero Remove the check for refcount already zero, which shouldn't be necessary. Change-Id: I0bc3966e21f8695f4787c2482dd5882dcd0f6dd9 Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 666c876..2d8647d 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -111,8 +111,6 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) } mutex_lock(&mapping->mutex); - if (atomic_read(&mapping->refcount.refcount) == 0) - dev_err(mapping->device, "Refcount is already 0\n"); if (kref_put(&mapping->refcount, release_entry)) { dev_dbg(mapping->device, "Removing Gasket sysfs mapping\n"); /* From f10af4ee11fa7141aa2af49544af5ddec902006f Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Fri, 27 Jul 2018 22:21:57 -0700 Subject: [PATCH 087/209] staging: gasket: apex: fixup undefined PCI class Apex chips with class 0 (PCI_CLASS_NOT_DEFINED) fixed up to PCI_CLASS_SYSTEM_OTHER to enable PCI resource assignments. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apex_driver.c b/apex_driver.c index 7a2646a..2001464 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -809,3 +809,10 @@ static ssize_t sysfs_show( gasket_sysfs_put_device_data(device, gasket_dev); return ret; } + +static void apex_pci_fixup_class(struct pci_dev *pdev) +{ + pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class; +} +DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, + PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); From a7ac1d2090d8b5d5c7327b16ba58c948a6feccd1 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Fri, 27 Jul 2018 22:21:58 -0700 Subject: [PATCH 088/209] staging: gasket: sysfs: remove unnecessary NULL check on device ptr The device pointer passed into get_mapping() will never be NULL; the check is unnecessary. Reported-by: Greg Kroah-Hartman Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 2d8647d..da972ce 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -67,11 +67,6 @@ static struct gasket_sysfs_mapping *get_mapping(struct device *device) { int i; - if (!device) { - pr_debug("%s: Received NULL device\n", __func__); - return NULL; - } - for (i = 0; i < GASKET_SYSFS_NUM_MAPPINGS; i++) { mutex_lock(&dev_mappings[i].mutex); if (dev_mappings[i].device == device) { From 7ad8140be71fe67823f0fa9e2a5668af6dce8427 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Fri, 27 Jul 2018 22:21:59 -0700 Subject: [PATCH 089/209] staging: gasket: page table: remove code for "no dma_ops" Remove code with TODOs on it for working around apparent problems previously seen in a qemu environment where dma_ops was not set correctly. There is no user of this in the current code. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 +- gasket_page_table.c | 58 +++++---------------------------------------- gasket_page_table.h | 3 +-- 3 files changed, 8 insertions(+), 55 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 6beee37..eba723e 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -926,7 +926,7 @@ static int gasket_enable_dev( &gasket_dev->bar_data[ driver_desc->page_table_bar_index], &driver_desc->page_table_configs[tbl_idx], - gasket_dev->dev, gasket_dev->pci_dev, true); + gasket_dev->dev, gasket_dev->pci_dev); if (ret) { dev_err(gasket_dev->dev, "Couldn't init page table %d: %d\n", diff --git a/gasket_page_table.c b/gasket_page_table.c index 32f1c1e..7228396 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -212,12 +212,6 @@ struct gasket_page_table { * gasket_mmap function, so user_virt belongs in the driver anyhow. */ struct gasket_coherent_page_entry *coherent_pages; - - /* - * Whether the page table uses arch specific dma_ops or - * whether the driver is supplying its own. - */ - bool dma_ops; }; /* Mapping declarations */ @@ -290,7 +284,7 @@ int gasket_page_table_init( struct gasket_page_table **ppg_tbl, const struct gasket_bar_data *bar_data, const struct gasket_page_table_config *page_table_config, - struct device *device, struct pci_dev *pci_dev, bool has_dma_ops) + struct device *device, struct pci_dev *pci_dev) { ulong bytes; struct gasket_page_table *pg_tbl; @@ -353,7 +347,6 @@ int gasket_page_table_init( bar_data->virt_base[page_table_config->extended_reg]); pg_tbl->device = device; pg_tbl->pci_dev = pci_dev; - pg_tbl->dma_ops = has_dma_ops; dev_dbg(device, "Page table initialized successfully\n"); @@ -759,33 +752,6 @@ static int gasket_map_extended_pages( return 0; } -/* - * TODO: dma_map_page() is not plugged properly when running under qemu. i.e. - * dma_ops are not set properly, which causes the kernel to assert. - * - * This temporary hack allows the driver to work on qemu, but need to be fixed: - * - either manually set the dma_ops for the architecture (which incidentally - * can't be done in an out-of-tree module) - or get qemu to fill the device tree - * properly so as linux plug the proper dma_ops or so as the driver can detect - * that it is runnig on qemu - */ -static inline dma_addr_t _no_op_dma_map_page( - struct device *dev, struct page *page, size_t offset, size_t size, - enum dma_data_direction dir) -{ - /* - * struct dma_map_ops *ops = get_dma_ops(dev); - * dma_addr_t addr; - * - * kmemcheck_mark_initialized(page_address(page) + offset, size); - * BUG_ON(!valid_dma_direction(dir)); - * addr = ops->map_page(dev, page, offset, size, dir, NULL); - * debug_dma_map_page(dev, page, offset, size, dir, addr, false); - */ - - return page_to_phys(page); -} - /* * Get and map last level page table buffers. * @pg_tbl: Gasket page table pointer. @@ -856,16 +822,9 @@ static int gasket_perform_mapping( ptes[i].offset = offset; /* Map the page into DMA space. */ - if (pg_tbl->dma_ops) { - /* hook in to kernel map functions */ - ptes[i].dma_addr = dma_map_page(pg_tbl->device, - page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); - } else { - ptes[i].dma_addr = _no_op_dma_map_page( - pg_tbl->device, page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); - } - + ptes[i].dma_addr = + dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); dev_dbg(pg_tbl->device, "%s i %d pte %p pfn %p -> mapped %llx\n", __func__, i, &ptes[i], @@ -1042,13 +1001,8 @@ static int gasket_alloc_extended_subtable( } /* Map the page into DMA space. */ - if (pg_tbl->dma_ops) { - pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, - PAGE_SIZE, DMA_BIDIRECTIONAL); - } else { - pte->dma_addr = _no_op_dma_map_page(pg_tbl->device, pte->page, - 0, PAGE_SIZE, DMA_BIDIRECTIONAL); - } + pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); /* Wait until the page is mapped. */ mb(); diff --git a/gasket_page_table.h b/gasket_page_table.h index 0e8afdb..7655886 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -37,7 +37,6 @@ struct gasket_page_table; * translation table. * @device: Device structure for the underlying device. Only used for logging. * @pci_dev: PCI system descriptor for the underlying device. - * @bool has_dma_ops: Whether the page table uses arch specific dma_ops or * whether the driver will supply its own. * * Description: Allocates and initializes data to track address translation - @@ -51,7 +50,7 @@ int gasket_page_table_init( struct gasket_page_table **ppg_tbl, const struct gasket_bar_data *bar_data, const struct gasket_page_table_config *page_table_config, - struct device *device, struct pci_dev *pci_dev, bool dma_ops); + struct device *device, struct pci_dev *pci_dev); /* * Deallocate and cleanup page table data. From 90e45919f417f6f5cf4c2b7cf103abdf3fa9c04d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sat, 28 Jul 2018 12:33:24 -0700 Subject: [PATCH 090/209] staging: gasket: core: hold reference on device while in use Hold a reference on the struct device while a pointer to that device is in use by gasket. Reported-by: Greg Kroah-Hartman Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index eba723e..6c81459 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -448,7 +448,7 @@ static int gasket_alloc_dev( gasket_dev->internal_desc = internal_desc; gasket_dev->dev_idx = dev_idx; snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", kobj_name); - gasket_dev->dev = parent; + gasket_dev->dev = get_device(parent); /* gasket_bar_data is uninitialized. */ gasket_dev->num_page_tables = driver_desc->num_page_tables; /* max_page_table_size and *page table are uninit'ed */ @@ -487,7 +487,7 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) mutex_lock(&internal_desc->mutex); internal_desc->devs[gasket_dev->dev_idx] = NULL; mutex_unlock(&internal_desc->mutex); - + put_device(gasket_dev->dev); kfree(gasket_dev); } From 1b13804ed55e13f44764a6e9f416bf612069a81a Mon Sep 17 00:00:00 2001 From: Dmitriy Cherkasov Date: Sat, 28 Jul 2018 22:55:24 +0000 Subject: [PATCH 091/209] staging: gasket: use NULL instead of 0 for null pointer Fixes sparse warning: Using plain integer as NULL pointer Signed-off-by: Dmitriy Cherkasov Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 7228396..b9304d2 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1605,7 +1605,7 @@ nomem: if (gasket_dev->page_table[index]->coherent_pages) { kfree(gasket_dev->page_table[index]->coherent_pages); - gasket_dev->page_table[index]->coherent_pages = 0; + gasket_dev->page_table[index]->coherent_pages = NULL; } gasket_dev->page_table[index]->num_coherent_pages = 0; return -ENOMEM; From 50ae685ab3aafa854ec61b236574eb07baa4f2a5 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:34 -0700 Subject: [PATCH 092/209] staging: gasket: core: hold reference to pci_dev while used Hold a reference on the struct pci_dev while a pointer to it is held in the gasket data structures. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 6c81459..b90726f 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -488,6 +488,7 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) internal_desc->devs[gasket_dev->dev_idx] = NULL; mutex_unlock(&internal_desc->mutex); put_device(gasket_dev->dev); + pci_dev_put(gasket_dev->pci_dev); kfree(gasket_dev); } @@ -565,6 +566,7 @@ static int gasket_pci_probe( ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); if (ret) return ret; + gasket_dev->pci_dev = pci_dev_get(pci_dev); if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { pr_err("Cannot create %s device %s [ret = %ld]\n", driver_desc->name, gasket_dev->dev_info.name, @@ -572,7 +574,6 @@ static int gasket_pci_probe( ret = -ENODEV; goto fail1; } - gasket_dev->pci_dev = pci_dev; ret = gasket_setup_pci(pci_dev, gasket_dev); if (ret) @@ -713,7 +714,6 @@ static int gasket_setup_pci( { int i, mapped_bars, ret; - gasket_dev->pci_dev = pci_dev; ret = pci_enable_device(pci_dev); if (ret) { dev_err(gasket_dev->dev, "cannot enable PCI device\n"); From c5505ad15cc881905f2dd4d6fb62cfb55cbdcd95 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:35 -0700 Subject: [PATCH 093/209] staging: gasket: sysfs: hold reference to device while in use Hold a reference to the struct device while a gasket sysfs mapping exists for the device and a pointer to the struct is kept in the mapping data structures. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index da972ce..fde0465 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -126,6 +126,7 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) kfree(mapping->attributes); mapping->attributes = NULL; mapping->attribute_count = 0; + put_device(mapping->device); mapping->device = NULL; mapping->gasket_dev = NULL; } @@ -208,22 +209,20 @@ int gasket_sysfs_create_mapping( device->kobj.name); mapping = &dev_mappings[map_idx]; - kref_init(&mapping->refcount); - mapping->device = device; - mapping->gasket_dev = gasket_dev; mapping->attributes = kcalloc(GASKET_SYSFS_MAX_NODES, sizeof(*mapping->attributes), GFP_KERNEL); - mapping->attribute_count = 0; if (!mapping->attributes) { dev_dbg(device, "Unable to allocate sysfs attribute array\n"); - mapping->device = NULL; - mapping->gasket_dev = NULL; mutex_unlock(&mapping->mutex); mutex_unlock(&function_mutex); return -ENOMEM; } + kref_init(&mapping->refcount); + mapping->device = get_device(device); + mapping->gasket_dev = gasket_dev; + mapping->attribute_count = 0; mutex_unlock(&mapping->mutex); mutex_unlock(&function_mutex); From 8719d43b7b89c4287aabc9dd2a18dea44b99f64f Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:36 -0700 Subject: [PATCH 094/209] staging: gasket: page table: hold references to device and pci_dev Hold references to the struct device and the pci_dev for the page table while the data structures contian pointers to these. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index b9304d2..6b946a1 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -345,8 +345,8 @@ int gasket_page_table_init( bar_data->virt_base[page_table_config->base_reg]); pg_tbl->extended_offset_reg = (u64 __iomem *)&( bar_data->virt_base[page_table_config->extended_reg]); - pg_tbl->device = device; - pg_tbl->pci_dev = pci_dev; + pg_tbl->device = get_device(device); + pg_tbl->pci_dev = pci_dev_get(pci_dev); dev_dbg(device, "Page table initialized successfully\n"); @@ -364,6 +364,8 @@ void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl) vfree(pg_tbl->entries); pg_tbl->entries = NULL; + put_device(pg_tbl->device); + pci_dev_put(pg_tbl->pci_dev); kfree(pg_tbl); } From e7585bf384e2973b10b1ebd86d1aafe6169fa7b4 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:37 -0700 Subject: [PATCH 095/209] staging: gasket: core: allow root access based on user namespace Use user namespace to determine whether gasket device file opener is root, allowing root access to containers, if necessary. Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index b90726f..4632607 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -13,13 +13,16 @@ #include "gasket_page_table.h" #include "gasket_sysfs.h" +#include #include #include #include #include #include #include +#include #include +#include #ifdef GASKET_KERNEL_TRACE_SUPPORT #define CREATE_TRACE_POINTS @@ -1074,7 +1077,8 @@ static int gasket_open(struct inode *inode, struct file *filp) char task_name[TASK_COMM_LEN]; struct gasket_cdev_info *dev_info = container_of(inode->i_cdev, struct gasket_cdev_info, cdev); - int is_root = capable(CAP_SYS_ADMIN); + struct pid_namespace *pid_ns = task_active_pid_ns(current); + int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; @@ -1157,6 +1161,8 @@ static int gasket_release(struct inode *inode, struct file *file) char task_name[TASK_COMM_LEN]; struct gasket_cdev_info *dev_info = container_of(inode->i_cdev, struct gasket_cdev_info, cdev); + struct pid_namespace *pid_ns = task_active_pid_ns(current); + int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; @@ -1168,7 +1174,7 @@ static int gasket_release(struct inode *inode, struct file *file) "Releasing device node. Call origin: tgid %u (%s) " "(f_mode: 0%03o, fmode_write: %d, is_root: %u)\n", current->tgid, task_name, file->f_mode, - (file->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN)); + (file->f_mode & FMODE_WRITE), is_root); dev_dbg(gasket_dev->dev, "Current open count (owning tgid %u): %d\n", ownership->owner, ownership->write_open_count); From 5193ebb0e40ce6383df3f13258a3e9cd4c13491c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:38 -0700 Subject: [PATCH 096/209] staging: gasket: apex: simplify comments for static functions Static functions don't need kernel doc formatting, can be simplified. Reformat comments that can be single-line. Remove extraneous text. Change-Id: Ia29594eeb2dd6cea38665d26e130237f5f90c94a Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 70 ++++++++------------------------------------------- 1 file changed, 11 insertions(+), 59 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 2001464..899f7d3 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -380,34 +380,20 @@ static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) gasket_dev->dev_info.device, apex_sysfs_attrs); } -/* On device open, we want to perform a core reinit reset. */ +/* On device open, perform a core reinit reset. */ static int apex_device_open_cb(struct gasket_dev *gasket_dev) { return gasket_reset_nolock(gasket_dev, APEX_CHIP_REINIT_RESET); } -/** - * apex_get_status - Set device status. - * @dev: Apex device struct. - * - * Description: Check the device status registers and set the driver status - * to ALIVE or DEAD. - * - * Returns 0 if status is ALIVE, a negative error number otherwise. - */ +/* Check the device status registers and return device status ALIVE or DEAD. */ static int apex_get_status(struct gasket_dev *gasket_dev) { /* TODO: Check device status. */ return GASKET_STATUS_ALIVE; } -/** - * apex_device_cleanup - Clean up Apex HW after close. - * @gasket_dev: Gasket device pointer. - * - * Description: Resets the Apex hardware. Called on final close via - * device_close_cb. - */ +/* Reset the Apex hardware. Called on final close via device_close_cb. */ static int apex_device_cleanup(struct gasket_dev *gasket_dev) { u64 scalar_error; @@ -431,14 +417,7 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) return ret; } -/** - * apex_reset - Quits reset. - * @gasket_dev: Gasket device pointer. - * - * Description: Resets the hardware, then quits reset. - * Called on device open. - * - */ +/* Reset the hardware, then quit reset. Called on device open. */ static int apex_reset(struct gasket_dev *gasket_dev, uint type) { int ret; @@ -461,9 +440,7 @@ static int apex_reset(struct gasket_dev *gasket_dev, uint type) return ret; } -/* - * Enters GCB reset state. - */ +/* Enter GCB reset state. */ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) { if (bypass_top_level) @@ -518,9 +495,7 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) return 0; } -/* - * Quits GCB reset state. - */ +/* Quit GCB reset state. */ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) { u32 val0, val1; @@ -603,9 +578,7 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) return 0; } -/* - * Determines if GCB is in reset state. - */ +/* Determine if GCB is in reset state. */ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) { u32 val = gasket_dev_read_32( @@ -617,9 +590,6 @@ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) /* * Check permissions for Apex ioctls. - * @file: File pointer from ioctl. - * @cmd: ioctl command. - * * Returns true if the current user may execute this ioctl, and false otherwise. */ static bool apex_ioctl_check_permissions(struct file *filp, uint cmd) @@ -627,9 +597,7 @@ static bool apex_ioctl_check_permissions(struct file *filp, uint cmd) return !!(filp->f_mode & FMODE_WRITE); } -/* - * Apex-specific ioctl handler. - */ +/* Apex-specific ioctl handler. */ static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) { struct gasket_dev *gasket_dev = filp->private_data; @@ -647,11 +615,7 @@ static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) } } -/* - * Gates or un-gates Apex clock. - * @gasket_dev: Gasket device pointer. - * @argp: User ioctl arg, pointer to a apex_gate_clock_ioctl struct. - */ +/* Gates or un-gates Apex clock. */ static long apex_clock_gating(struct gasket_dev *gasket_dev, struct apex_gate_clock_ioctl __user *argp) { @@ -685,11 +649,7 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, return 0; } -/* - * apex_set_performance_expectation: Adjust clock rates for Apex. - * @gasket_dev: device pointer. - * @arg: User ioctl arg, in this case to an apex_performance_expectation_ioctl struct. - */ +/* apex_set_performance_expectation: Adjust clock rates for Apex. */ static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulong arg) { struct apex_performance_expectation_ioctl ibuf; @@ -751,15 +711,7 @@ static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulon return 0; } -/* - * Display driver sysfs entries. - * @device: Kernel device structure. - * @attr: Attribute to display. - * @buf: Buffer to which to write output. - * - * Description: Looks up the driver data and file-specific attribute data (the - * type of the attribute), then fills "buf" accordingly. - */ +/* Display driver sysfs entries. */ static ssize_t sysfs_show( struct device *device, struct device_attribute *attr, char *buf) { From 8393e3f10d00bb5013d8d2b358df9a6e3dcd2509 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:39 -0700 Subject: [PATCH 097/209] staging: gasket: core: simplify comments for static functions Static functions don't need kernel doc formatting, can be simplified. Reformat comments that can be single-line. Remove extraneous text. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 151 +++++++++++--------------------------------------- 1 file changed, 31 insertions(+), 120 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 4632607..3aa6a21 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -199,11 +199,7 @@ MODULE_AUTHOR("Rob Springer "); module_init(gasket_init); module_exit(gasket_exit); -/* - * Perform a standard Gasket callback. - * @gasket_dev: Device specific pointer to forward. - * @cb_function: Standard callback to perform. - */ +/* Perform a standard Gasket callback. */ static inline int check_and_invoke_callback( struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) { @@ -219,13 +215,7 @@ static inline int check_and_invoke_callback( return ret; } -/* - * Perform a standard Gasket callback - * without grabbing gasket_dev->mutex. - * @gasket_dev: Device specific pointer to forward. - * @cb_function: Standard callback to perform. - * - */ +/* Perform a standard Gasket callback without grabbing gasket_dev->mutex. */ static inline int gasket_check_and_invoke_callback_nolock( struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) { @@ -240,9 +230,8 @@ static inline int gasket_check_and_invoke_callback_nolock( } /* - * Returns nonzero if the gasket_cdev_info is owned by the current thread group + * Return nonzero if the gasket_cdev_info is owned by the current thread group * ID. - * @info: Device node info. */ static int gasket_owned_by_current_tgid(struct gasket_cdev_info *info) { @@ -410,14 +399,9 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) } EXPORT_SYMBOL(gasket_unregister_device); -/** - * Allocate a Gasket device. - * @internal_desc: Pointer to the internal data for the device driver. - * @pdev: Pointer to the Gasket device pointer, the allocated device. - * @kobj_name: PCIe name for the device - * - * Description: Allocates and initializes a Gasket device structure. - * Adds the device to the device list. +/* + * Allocate and initialize a Gasket device structure, add the device to the + * device list. * * Returns 0 if successful, a negative error code otherwise. */ @@ -476,13 +460,7 @@ static int gasket_alloc_dev( return 0; } -/* - * Free a Gasket device. - * @internal_dev: Gasket device pointer; the device to unregister and free. - * - * Description: Removes the device from the device list and frees - * the Gasket device structure. - */ +/* Free a Gasket device. */ static void gasket_free_dev(struct gasket_dev *gasket_dev) { struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc; @@ -496,7 +474,7 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) } /* - * Finds the next free gasket_internal_dev slot. + * Find the next free gasket_internal_dev slot. * * Returns the located slot number on success or a negative number on failure. */ @@ -533,10 +511,8 @@ static int gasket_find_dev_slot( return i; } -/** +/* * PCI subsystem probe function. - * @pci_dev: PCI device pointer to the new device. - * @id: PCI device id structure pointer, the vendor and device ids. * * Called when a Gasket device is found. Allocates device metadata, maps device * memory, and calls gasket_enable_dev to prepare the device for active use. @@ -651,7 +627,6 @@ fail1: /* * PCI subsystem remove function. - * @pci_dev: PCI device pointer; the device to remove. * * Called to remove a Gasket device. Finds the device in the device list and * cleans up metadata. @@ -704,8 +679,6 @@ static void gasket_pci_remove(struct pci_dev *pci_dev) /* * Setup PCI & set up memory mapping for the specified device. - * @pci_dev: pointer to the particular PCI device. - * @internal_dev: Corresponding Gasket device pointer. * * Enables the PCI device, reads the BAR registers and sets up pointers to the * device's memory mapped IO space. @@ -756,8 +729,6 @@ static void gasket_cleanup_pci(struct gasket_dev *gasket_dev) /* * Maps the specified bar into kernel space. - * @internal_dev: Device possessing the BAR to map. - * @bar_num: The BAR to map. * * Returns 0 on success, a negative error code otherwise. * A zero-sized BAR will not be mapped, but is not an error. @@ -834,7 +805,6 @@ fail: /* * Releases PCI BAR mapping. - * @internal_dev: Device possessing the BAR to unmap. * * A zero-sized or not-mapped BAR will not be unmapped, but is not an error. */ @@ -866,12 +836,7 @@ static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num) release_mem_region(base, bytes); } -/* - * Handle adding a char device and related info. - * @dev_info: Pointer to the dev_info struct for this device. - * @file_ops: The file operations for this device. - * @owner: The owning module for this device. - */ +/* Add a char device and related info. */ static int gasket_add_cdev( struct gasket_cdev_info *dev_info, const struct file_operations *file_ops, struct module *owner) @@ -891,14 +856,7 @@ static int gasket_add_cdev( return 0; } -/* - * Performs final init and marks the device as active. - * @internal_desc: Pointer to Gasket [internal] driver descriptor structure. - * @internal_dev: Pointer to Gasket [internal] device structure. - * - * Currently forwards all work to device-specific callback; a future phase will - * extract elements of character device registration here. - */ +/* Perform final init and marks the device as active. */ static int gasket_enable_dev( struct gasket_internal_desc *internal_desc, struct gasket_dev *gasket_dev) @@ -976,13 +934,7 @@ static int gasket_enable_dev( return 0; } -/* - * Disable device operations. - * @gasket_dev: Pointer to Gasket device structure. - * - * Currently forwards all work to device-specific callback; a future phase will - * extract elements of character device unregistration here. - */ +/* Disable device operations. */ static void gasket_disable_dev(struct gasket_dev *gasket_dev) { const struct gasket_driver_desc *driver_desc = @@ -1007,7 +959,7 @@ static void gasket_disable_dev(struct gasket_dev *gasket_dev) check_and_invoke_callback(gasket_dev, driver_desc->disable_dev_cb); } -/** +/* * Registered descriptor lookup. * * Precondition: Called with g_mutex held (to avoid a race on return). @@ -1055,18 +1007,15 @@ const char *gasket_num_name_lookup( } EXPORT_SYMBOL(gasket_num_name_lookup); -/** - * Opens the char device file. - * @inode: Inode structure pointer of the device file. - * @file: File structure pointer. +/* + * Open the char device file. * - * Description: Called on an open of the device file. If the open is for - * writing, and the device is not owned, this process becomes - * the owner. If the open is for writing and the device is - * already owned by some other process, it is an error. If - * this process is the owner, increment the open count. + * If the open is for writing, and the device is not owned, this process becomes + * the owner. If the open is for writing and the device is already owned by + * some other process, it is an error. If this process is the owner, increment + * the open count. * - * Returns 0 if successful, a negative error number otherwise. + * Returns 0 if successful, a negative error number otherwise. */ static int gasket_open(struct inode *inode, struct file *filp) { @@ -1140,17 +1089,12 @@ static int gasket_open(struct inode *inode, struct file *filp) return 0; } -/** - * gasket_release - Close of the char device file. - * @inode: Inode structure pointer of the device file. - * @file: File structure pointer. +/* + * Called on a close of the device file. If this process is the owner, + * decrement the open count. On last close by the owner, free up buffers and + * eventfd contexts, and release ownership. * - * Description: Called on a close of the device file. If this process - * is the owner, decrement the open count. On last close - * by the owner, free up buffers and eventfd contexts, and - * release ownership. - * - * Returns 0 if successful, a negative error number otherwise. + * Returns 0 if successful, a negative error number otherwise. */ static int gasket_release(struct inode *inode, struct file *file) { @@ -1209,10 +1153,6 @@ static int gasket_release(struct inode *inode, struct file *file) } /* - * Permission and validity checking for mmap ops. - * @gasket_dev: Gasket device information structure. - * @vma: Standard virtual memory area descriptor. - * * Verifies that the user has permissions to perform the requested mapping and * that the provided descriptor/range is of adequate size to hold the range to * be mapped. @@ -1256,11 +1196,6 @@ static bool gasket_mmap_has_permissions( } /* - * Checks if an address is within the region - * allocated for coherent buffer. - * @driver_desc: driver description. - * @address: offset of address to check. - * * Verifies that the input address is within the region allocated to coherent * buffer. */ @@ -1512,11 +1447,7 @@ static int gasket_mm_vma_bar_offset( return 0; } -/* - * Map a region of coherent memory. - * @gasket_dev: Gasket device handle. - * @vma: Virtual memory area descriptor with region to map. - */ +/* Map a region of coherent memory. */ static int gasket_mmap_coherent( struct gasket_dev *gasket_dev, struct vm_area_struct *vma) { @@ -1561,16 +1492,7 @@ static int gasket_mmap_coherent( return 0; } -/* - * Maps a device's BARs into user space. - * @filp: File structure pointer describing this node usage session. - * @vma: Standard virtual memory area descriptor. - * - * Maps the entirety of each of the device's BAR ranges into the user memory - * range specified by vma. - * - * Returns 0 on success, a negative errno on error. - */ +/* Map a device's BARs into user space. */ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) { int i, ret; @@ -1714,14 +1636,7 @@ fail: return ret; } -/* - * Determine the health of the Gasket device. - * @gasket_dev: Gasket device structure. - * - * Checks the underlying device health (via the device_status_cb) - * and the status of initialized Gasket code systems (currently - * only interrupts), then returns a gasket_status appropriately. - */ +/* Determine the health of the Gasket device. */ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) { int status; @@ -1760,14 +1675,10 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) /* * Gasket ioctl dispatch function. - * @filp: File structure pointer describing this node usage session. - * @cmd: ioctl number to handle. - * @arg: ioctl-specific data pointer. * - * First, checks if the ioctl is a generic ioctl. If not, it passes - * the ioctl to the ioctl_handler_cb registered in the driver description. - * If the ioctl is a generic ioctl, the function passes it to the - * gasket_ioctl_handler in gasket_ioctl.c. + * Check if the ioctl is a generic ioctl. If not, pass the ioctl to the + * ioctl_handler_cb registered in the driver description. + * If the ioctl is a generic ioctl, pass it to gasket_ioctl_handler. */ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) { From 393f9db42f0f70d435fe8ac1482b25174513d174 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:40 -0700 Subject: [PATCH 098/209] staging: gasket: ioctl: simplify comments for static functions Static functions don't need kernel doc formatting, can be simplified. Reformat comments that can be single-line. Remove extraneous text. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 51 +++++++++----------------------------------------- 1 file changed, 9 insertions(+), 42 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 78a132a..55bdd7b 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -170,14 +170,7 @@ long gasket_is_supported_ioctl(uint cmd) } } -/* - * Permission checker for Gasket ioctls. - * @filp: File structure pointer describing this node usage session. - * @cmd: ioctl number to handle. - * - * Check permissions for Gasket ioctls. - * Returns true if the file opener may execute this ioctl, or false otherwise. - */ +/* Check permissions for Gasket ioctls. */ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) { bool alive; @@ -218,11 +211,7 @@ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) return false; /* unknown permissions */ } -/* - * Associate an eventfd with an interrupt. - * @gasket_dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to gasket_interrupt_eventfd struct in userspace. - */ +/* Associate an eventfd with an interrupt. */ static int gasket_set_event_fd(struct gasket_dev *gasket_dev, struct gasket_interrupt_eventfd __user *argp) { @@ -237,11 +226,7 @@ static int gasket_set_event_fd(struct gasket_dev *gasket_dev, gasket_dev->interrupt_data, die.interrupt, die.event_fd); } -/* - * Reads the size of the page table. - * @gasket_dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to gasket_page_table_ioctl struct in userspace. - */ +/* Read the size of the page table. */ static int gasket_read_page_table_size( struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) @@ -268,11 +253,7 @@ static int gasket_read_page_table_size( return ret; } -/* - * Reads the size of the simple page table. - * @gasket_dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to gasket_page_table_ioctl struct in userspace. - */ +/* Read the size of the simple page table. */ static int gasket_read_simple_page_table_size( struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) @@ -299,11 +280,7 @@ static int gasket_read_simple_page_table_size( return ret; } -/* - * Sets the boundary between the simple and extended page tables. - * @gasket_dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to gasket_page_table_ioctl struct in userspace. - */ +/* Set the boundary between the simple and extended page tables. */ static int gasket_partition_page_table( struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) @@ -340,11 +317,7 @@ static int gasket_partition_page_table( return ret; } -/* - * Maps a userspace buffer to a device virtual address. - * @gasket_dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to a gasket_page_table_ioctl struct in userspace. - */ +/* Map a userspace buffer to a device virtual address. */ static int gasket_map_buffers(struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) { @@ -370,11 +343,7 @@ static int gasket_map_buffers(struct gasket_dev *gasket_dev, ibuf.host_address, ibuf.device_address, ibuf.size / PAGE_SIZE); } -/* - * Unmaps a userspace buffer from a device virtual address. - * @gasket_dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to a gasket_page_table_ioctl struct in userspace. - */ +/* Unmap a userspace buffer from a device virtual address. */ static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) { @@ -402,10 +371,8 @@ static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, } /* - * Tell the driver to reserve structures for coherent allocation, and allocate - * or free the corresponding memory. - * @dev: Pointer to the current gasket_dev we're using. - * @argp: Pointer to a gasket_coherent_alloc_config_ioctl struct in userspace. + * Reserve structures for coherent allocation, and allocate or free the + * corresponding memory. */ static int gasket_config_coherent_allocator( struct gasket_dev *gasket_dev, From 710158f4ca862d31690a881f1e9b527df91e1c5b Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:41 -0700 Subject: [PATCH 099/209] staging: gasket: page table: simplify comments for static functions Static functions don't need kernel doc formatting, can be simplified. Reformat comments that can be single-line. Remove extraneous text. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 323 +++++++------------------------------------- 1 file changed, 48 insertions(+), 275 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 6b946a1..b42f663 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -635,27 +635,9 @@ int gasket_page_table_system_status(struct gasket_page_table *page_table) return GASKET_STATUS_ALIVE; } -/* Internal functions */ - -/* Mapping functions */ /* * Allocate and map pages to simple addresses. - * @pg_tbl: Gasket page table pointer. - * @host_addr: Starting host virtual memory address of the pages. - * @dev_addr: Starting device address of the pages. - * @cnt: Count of the number of device pages to map. - * - * Description: gasket_map_simple_pages calls gasket_simple_alloc_pages() to - * allocate the page table slots, then calls - * gasket_perform_mapping() to actually do the work of mapping the - * pages into the the simple page table (device translation table - * registers). - * - * The sd_mutex must be held when gasket_map_simple_pages() is - * called. - * - * Returns 0 if successful or a non-zero error number otherwise. - * If there is an error, no pages are mapped. + * If there is an error, no pages are mapped. */ static int gasket_map_simple_pages( struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, @@ -685,22 +667,7 @@ static int gasket_map_simple_pages( /* * gasket_map_extended_pages - Get and map buffers to extended addresses. - * @pg_tbl: Gasket page table pointer. - * @host_addr: Starting host virtual memory address of the pages. - * @dev_addr: Starting device address of the pages. - * @num_pages: The number of device pages to map. - * - * Description: gasket_map_extended_buffers calls - * gasket_alloc_extended_entries() to allocate the page table - * slots, then loops over the level 0 page table entries, and for - * each calls gasket_perform_mapping() to map the buffers into the - * level 1 page table for that level 0 entry. - * - * The page table mutex must be held when - * gasket_map_extended_pages() is called. - * - * Returns 0 if successful or a non-zero error number otherwise. - * If there is an error, no pages are mapped. + * If there is an error, no pages are mapped. */ static int gasket_map_extended_pages( struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, @@ -756,32 +723,11 @@ static int gasket_map_extended_pages( /* * Get and map last level page table buffers. - * @pg_tbl: Gasket page table pointer. - * @ptes: Array of page table entries to describe this mapping, one per - * page to map. - * @slots: Location(s) to write device-mapped page address. If this is a simple - * mapping, these will be address translation registers. If this is - * an extended mapping, these will be within a second-level page table - * allocated by the host and so must have their __iomem attribute - * casted away. - * @host_addr: Starting [host] virtual memory address of the buffers. - * @num_pages: The number of device pages to map. - * @is_simple_mapping: 1 if this is a simple mapping, 0 otherwise. * - * Description: gasket_perform_mapping calls get_user_pages() to get pages - * of user memory and pin them. It then calls dma_map_page() to - * map them for DMA. Finally, the mapped DMA addresses are written - * into the page table. - * - * This function expects that the page table entries are - * already allocated. The level argument determines how the - * final page table entries are written: either into PCIe memory - * mapped space for a level 0 page table or into kernel memory - * for a level 1 page table. - * - * The page pointers are saved for later releasing the pages. - * - * Returns 0 if successful or a non-zero error number otherwise. + * slots is the location(s) to write device-mapped page address. If this is a + * simple mapping, these will be address translation registers. If this is + * an extended mapping, these will be within a second-level page table + * allocated by the host and so must have their __iomem attribute casted away. */ static int gasket_perform_mapping( struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, @@ -866,21 +812,9 @@ static int gasket_perform_mapping( return 0; } -/** +/* * Allocate page table entries in a simple table. - * @pg_tbl: Gasket page table pointer. - * @dev_addr: Starting device address for the (eventual) mappings. - * @num_pages: Count of pages to be mapped. - * - * Description: gasket_alloc_simple_entries checks to see if a range of page - * table slots are available. As long as the sd_mutex is - * held, the slots will be available. - * - * The page table mutex must be held when - * gasket_alloc_simple entries() is called. - * - * Returns 0 if successful, or non-zero if the requested device - * addresses are not available. + * The page table mutex must be held by the caller. */ static int gasket_alloc_simple_entries( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) @@ -893,29 +827,19 @@ static int gasket_alloc_simple_entries( return 0; } -/** - * Allocate slots in an extended page table. - * @pg_tbl: Gasket page table pointer. - * @dev_addr: Starting device address for the (eventual) mappings. - * @num_pages: Count of pages to be mapped. +/* + * Allocate slots in an extended page table. Check to see if a range of page + * table slots are available. If necessary, memory is allocated for second level + * page tables. * - * Description: gasket_alloc_extended_entries checks to see if a range of page - * table slots are available. If necessary, memory is allocated for - * second level page tables. + * Note that memory for second level page tables is allocated as needed, but + * that memory is only freed on the final close of the device file, when the + * page tables are repartitioned, or the the device is removed. If there is an + * error or if the full range of slots is not available, any memory + * allocated for second level page tables remains allocated until final close, + * repartition, or device removal. * - * Note that memory for second level page tables is allocated - * as needed, but that memory is only freed on the final close - * of the device file, when the page tables are repartitioned, - * or the the device is removed. If there is an error or if - * the full range of slots is not available, any memory - * allocated for second level page tables remains allocated - * until final close, repartition, or device removal. - * - * The page table mutex must be held when - * gasket_alloc_extended_entries() is called. - * - * Returns 0 if successful, or non-zero if the slots are - * not available. + * The page table mutex must be held by the caller. */ static int gasket_alloc_extended_entries( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries) @@ -958,21 +882,9 @@ static int gasket_alloc_extended_entries( return 0; } -/** +/* * Allocate a second level page table. - * @pg_tbl: Gasket page table pointer. - * @pte: Extended page table entry under/for which to allocate a second level. - * @slot: [Device] slot corresponding to pte. - * - * Description: Allocate the memory for a second level page table (subtable) at - * the given level 0 entry. Then call dma_map_page() to map the - * second level page table for DMA. Finally, write the - * mapped DMA address into the device page table. - * - * The page table mutex must be held when - * gasket_alloc_extended_subtable() is called. - * - * Returns 0 if successful, or a non-zero error otherwise. + * The page table mutex must be held by the caller. */ static int gasket_alloc_extended_subtable( struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, @@ -1017,15 +929,9 @@ static int gasket_alloc_extended_subtable( return 0; } -/* Unmapping functions */ /* * Non-locking entry to unmapping routines. - * @pg_tbl: Gasket page table structure. - * @dev_addr: Starting device address of the pages to unmap. - * @num_pages: The number of device pages to unmap. - * - * Description: Version of gasket_unmap_pages that assumes the page table lock - * is held. + * The page table mutex must be held by the caller. */ static void gasket_page_table_unmap_nolock( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) @@ -1041,14 +947,7 @@ static void gasket_page_table_unmap_nolock( /* * Unmap and release pages mapped to simple addresses. - * @pg_tbl: Gasket page table pointer. - * @dev_addr: Starting device address of the buffers. - * @num_pages: The number of device pages to unmap. - * - * Description: gasket_simple_unmap_pages calls gasket_perform_unmapping() to - * unmap and release the buffers in the level 0 page table. - * - * The sd_mutex must be held when gasket_unmap_simple_pages() is called. + * The page table mutex must be held by the caller. */ static void gasket_unmap_simple_pages( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) @@ -1059,20 +958,9 @@ static void gasket_unmap_simple_pages( pg_tbl->base_slot + slot, num_pages, 1); } -/** +/* * Unmap and release buffers to extended addresses. - * @pg_tbl: Gasket page table pointer. - * @dev_addr: Starting device address of the pages to unmap. - * @addr: Starting device address of the buffers. - * @num_pages: The number of device pages to unmap. - * - * Description: gasket_extended_unmap_pages loops over the level 0 page table - * entries, and for each calls gasket_perform_unmapping() to unmap - * the buffers from the level 1 page [sub]table for that level 0 - * entry. - * - * The page table mutex must be held when - * gasket_unmap_extended_pages() is called. + * The page table mutex must be held by the caller. */ static void gasket_unmap_extended_pages( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) @@ -1106,28 +994,7 @@ static void gasket_unmap_extended_pages( /* * Unmap and release mapped pages. - * @pg_tbl: Gasket page table pointer. - * @ptes: Array of page table entries to describe the mapped range, one per - * page to unmap. - * @slots: Device slots corresponding to the mappings described by "ptes". - * As with ptes, one element per page to unmap. - * If these are simple mappings, these will be address translation - * registers. If these are extended mappings, these will be witin a - * second-level page table allocated on the host, and so must have - * their __iomem attribute casted away. - * @num_pages: Number of pages to unmap. - * @is_simple_mapping: 1 if this is a simple mapping, 0 otherwise. - * - * Description: gasket_perform_unmapping() loops through the metadata entries - * in a last level page table (simple table or extended subtable), - * and for each page: - * - Unmaps the page from DMA space (dma_unmap_page), - * - Returns the page to the OS (gasket_release_page), - * The entry in the page table is written to 0. The metadata - * type is set to PTE_FREE and the metadata is all reset - * to 0. - * - * The page table mutex must be held when this function is called. + * The page table mutex must be held by the caller. */ static void gasket_perform_unmapping( struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, @@ -1165,17 +1032,6 @@ static void gasket_perform_unmapping( /* * Free a second level page [sub]table. - * @pg_tbl: Gasket page table pointer. - * @pte: Page table entry _pointing_to_ the subtable to free. - * @slot: Device slot holding a pointer to the sublevel's contents. - * - * Description: Safely deallocates a second-level [sub]table by: - * - Marking the containing first-level PTE as free - * - Setting the corresponding [extended] device slot as NULL - * - Unmapping the PTE from DMA space. - * - Freeing the subtable's memory. - * - Deallocating the page and clearing out the PTE. - * * The page table mutex must be held before this call. */ static void gasket_free_extended_subtable( @@ -1202,12 +1058,7 @@ static void gasket_free_extended_subtable( memset(pte, 0, sizeof(struct gasket_page_table_entry)); } -/* - * Safely return a page to the OS. - * @page: The page to return to the OS. - * Returns true if the page was released, false if it was - * ignored. - */ +/* Safely return a page to the OS. */ static bool gasket_release_page(struct page *page) { if (!page) @@ -1229,13 +1080,10 @@ static inline bool gasket_addr_is_simple( /* * Validity checking for simple addresses. - * @pg_tbl: Gasket page table pointer. - * @dev_addr: The device address to which the pages will be mapped. - * @num_pages: The number of pages in the range to consider. * - * Description: This call verifies that address translation commutes (from - * address to/from page + offset) and that the requested page range starts and - * ends within the set of currently-partitioned simple pages. + * Verify that address translation commutes (from address to/from page + offset) + * and that the requested page range starts and ends within the set of + * currently-partitioned simple pages. */ static bool gasket_is_simple_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) @@ -1269,13 +1117,11 @@ static bool gasket_is_simple_dev_addr_bad( } /* - * Verifies that address translation commutes (from address to/from page + - * offset) and that the requested page range starts and ends within the set of - * currently-partitioned simple pages. + * Validity checking for extended addresses. * - * @pg_tbl: Gasket page table pointer. - * @dev_addr: The device address to which the pages will be mapped. - * @num_pages: The number of second-level/sub pages in the range to consider. + * Verify that address translation commutes (from address to/from page + + * offset) and that the requested page range starts and ends within the set of + * currently-partitioned extended pages. */ static bool gasket_is_extended_dev_addr_bad( struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) @@ -1331,15 +1177,8 @@ static bool gasket_is_extended_dev_addr_bad( } /* - * Checks if a range of PTEs is free. - * @ptes: The set of PTEs to check. - * @num_entries: The number of PTEs to check. - * - * Description: Iterates over the input PTEs to determine if all have been - * marked as FREE or if any are INUSE. In the former case, 1/true is returned. - * Otherwise, 0/false is returned. - * - * The page table mutex must be held before this call. + * Check if a range of PTEs is free. + * The page table mutex must be held by the caller. */ static bool gasket_is_pte_range_free( struct gasket_page_table_entry *ptes, uint num_entries) @@ -1356,10 +1195,7 @@ static bool gasket_is_pte_range_free( /* * Actually perform collection. - * @pg_tbl: Gasket page table structure. - * - * Description: Version of gasket_page_table_garbage_collect that assumes the - * page table lock is held. + * The page table mutex must be held by the caller. */ static void gasket_page_table_garbage_collect_nolock( struct gasket_page_table *pg_tbl) @@ -1384,14 +1220,7 @@ static void gasket_page_table_garbage_collect_nolock( } /* - * Converts components to a device address. - * @pg_tbl: Gasket page table structure. - * @is_simple: nonzero if this should be a simple entry, zero otherwise. - * @page_index: The page index into the respective table. - * @offset: The offset within the requested page. - * - * Simple utility function to convert (simple, page, offset) into a device - * address. + * Convert (simple, page, offset) into a device address. * Examples: * Simple page 0, offset 32: * Input (0, 0, 32), Output 0x20 @@ -1429,14 +1258,7 @@ static ulong gasket_components_to_dev_address( } /* - * Gets the index of the address' page in the simple table. - * @pg_tbl: Gasket page table structure. - * @dev_addr: The address whose page index to retrieve. - * - * Description: Treats the input address as a simple address and determines the - * index of its underlying page in the simple page table (i.e., device address - * translation registers. - * + * Return the index of the page for the address in the simple table. * Does not perform validity checking. */ static int gasket_simple_page_idx( @@ -1447,14 +1269,7 @@ static int gasket_simple_page_idx( } /* - * Gets the level 0 page index for the given address. - * @pg_tbl: Gasket page table structure. - * @dev_addr: The address whose page index to retrieve. - * - * Description: Treats the input address as an extended address and determines - * the index of its underlying page in the first-level extended page table - * (i.e., device extended address translation registers). - * + * Return the level 0 page index for the given address. * Does not perform validity checking. */ static ulong gasket_extended_lvl0_page_idx( @@ -1465,14 +1280,7 @@ static ulong gasket_extended_lvl0_page_idx( } /* - * Gets the level 1 page index for the given address. - * @pg_tbl: Gasket page table structure. - * @dev_addr: The address whose page index to retrieve. - * - * Description: Treats the input address as an extended address and determines - * the index of its underlying page in the second-level extended page table - * (i.e., host memory pointed to by a first-level page table entry). - * + * Return the level 1 page index for the given address. * Does not perform validity checking. */ static ulong gasket_extended_lvl1_page_idx( @@ -1483,13 +1291,10 @@ static ulong gasket_extended_lvl1_page_idx( } /* - * Determines whether a host buffer was mapped as coherent memory. - * @pg_tbl: gasket_page_table structure tracking the host buffer mapping - * @host_addr: user virtual address within a host buffer + * Return whether a host buffer was mapped as coherent memory. * - * Description: A Gasket page_table currently support one contiguous - * dma range, mapped to one contiguous virtual memory range. Check if the - * host_addr is within start of page 0, and end of last page, for that range. + * A Gasket page_table currently support one contiguous dma range, mapped to one + * contiguous virtual memory range. Check if the host_addr is within that range. */ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) { @@ -1505,16 +1310,7 @@ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) return min <= host_addr && host_addr < max; } -/* - * Records the host_addr to coherent dma memory mapping. - * @gasket_dev: Gasket Device. - * @size: Size of the virtual address range to map. - * @dma_address: Dma address within the coherent memory range. - * @vma: Virtual address we wish to map to coherent memory. - * - * Description: For each page in the virtual address range, record the - * coherent page mgasket_pretapping. - */ +/* Record the host_addr to coherent dma memory mapping. */ int gasket_set_user_virt( struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address, ulong vma) @@ -1541,16 +1337,7 @@ int gasket_set_user_virt( return 0; } -/* - * Allocate a block of coherent memory. - * @gasket_dev: Gasket Device. - * @size: Size of the memory block. - * @dma_address: Dma address allocated by the kernel. - * @index: Index of the gasket_page_table within this Gasket device - * - * Description: Allocate a contiguous coherent memory block, DMA'ble - * by this device. - */ +/* Allocate a block of coherent memory. */ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, dma_addr_t *dma_address, u64 index) { @@ -1613,15 +1400,7 @@ nomem: return -ENOMEM; } -/* - * Free a block of coherent memory. - * @gasket_dev: Gasket Device. - * @size: Size of the memory block. - * @dma_address: Dma address allocated by the kernel. - * @index: Index of the gasket_page_table within this Gasket device - * - * Description: Release memory allocated thru gasket_alloc_coherent_memory. - */ +/* Free a block of coherent memory. */ int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address, u64 index) { @@ -1647,13 +1426,7 @@ int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, return 0; } -/* - * Release all coherent memory. - * @gasket_dev: Gasket Device. - * @index: Index of the gasket_page_table within this Gasket device - * - * Description: Release all memory allocated thru gasket_alloc_coherent_memory. - */ +/* Release all coherent memory. */ void gasket_free_coherent_memory_all( struct gasket_dev *gasket_dev, u64 index) { From 07367070d34b170650ead859613b992a55c67e4d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:42 -0700 Subject: [PATCH 100/209] staging: gasket: interrupt: simplify comments for static functions Static functions don't need kernel doc formatting, can be simplified. Reformat comments that can be single-line. Remove extraneous text. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 3be8e24..27fde99 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -87,13 +87,6 @@ static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = { GASKET_END_OF_ATTR_ARRAY, }; -/* - * Set up device registers for interrupt handling. - * @gasket_dev: The Gasket information structure for this device. - * - * Sets up the device registers with the correct indices for the relevant - * interrupts. - */ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev); /* MSIX init and cleanup. */ @@ -334,13 +327,7 @@ int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev) return 0; } -/* - * Set up device registers for interrupt handling. - * @gasket_dev: The Gasket information structure for this device. - * - * Sets up the device registers with the correct indices for the relevant - * interrupts. - */ +/* Set up device registers for interrupt handling. */ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) { int i; @@ -553,9 +540,6 @@ static ssize_t interrupt_sysfs_show( return ret; } -/* - * MSIX interrupt handler, used with PCI driver. - */ static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) { struct eventfd_ctx *ctx; From 2bf72399cac0cbbfe778533964c23e8155bd654c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:43 -0700 Subject: [PATCH 101/209] staging: gasket: sysfs: simplify comments for static functions Static functions don't need kernel doc formatting, can be simplified. Reformat comments that can be single-line. Remove extraneous text. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index fde0465..ef4eca0 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -47,22 +47,13 @@ struct gasket_sysfs_mapping { */ static struct gasket_sysfs_mapping dev_mappings[GASKET_SYSFS_NUM_MAPPINGS]; -/* - * Callback when a mapping's refcount goes to zero. - * @ref: The reference count of the containing sysfs mapping. - */ +/* Callback when a mapping's refcount goes to zero. */ static void release_entry(struct kref *ref) { /* All work is done after the return from kref_put. */ } -/* - * Looks up mapping information for the given device. - * @device: The device whose mapping to look for. - * - * Looks up the requested device and takes a reference and returns it if found, - * and returns NULL otherwise. - */ +/* Look up mapping information for the given device. */ static struct gasket_sysfs_mapping *get_mapping(struct device *device) { int i; @@ -82,17 +73,7 @@ static struct gasket_sysfs_mapping *get_mapping(struct device *device) return NULL; } -/* - * Returns a reference to a mapping. - * @mapping: The mapping we're returning. - * - * Decrements the refcount for the given mapping (if valid). If the refcount is - * zero, then it cleans up the mapping - in this function as opposed to the - * kref_put callback, due to a potential deadlock. - * - * Although put_mapping_n exists, this function is left here (as an implicit - * put_mapping_n(..., 1) for convenience. - */ +/* Put a reference to a mapping. */ static void put_mapping(struct gasket_sysfs_mapping *mapping) { int i; @@ -140,8 +121,7 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) } /* - * Returns a reference N times. - * @mapping: The mapping to return. + * Put a reference to a mapping N times. * * In higher-level resource acquire/release function pairs, the release function * will need to release a mapping 2x - once for the refcount taken in the From 0502a2e1c3b6e5738ea1def42686d4d9fe35c1bc Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:44 -0700 Subject: [PATCH 102/209] staging: gasket: TODO: remove entry for static function kernel docs Remove the TODO entry for simplifying kernel doc style comments for static functions, now that this has been addressed. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index fb71997..7f4c13c 100644 --- a/TODO +++ b/TODO @@ -5,7 +5,6 @@ staging directory. - Use misc interface instead of major number for driver version description. - Add descriptions of module_param's - apex_get_status() should actually check status. -- Static functions don't need kernel doc formatting, can be simplified. - Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); From 1694500340389600075750d6c92ae27c713904d7 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:45 -0700 Subject: [PATCH 103/209] staging: gasket: apex: remove static function forward declarations Remove forward declarations of static functions, move code to avoid forward references, for kernel style. Change-Id: I1078c4840860c5c1cf06a12bfd0409c6db7303ff Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 422 +++++++++++++++++++++++--------------------------- 1 file changed, 192 insertions(+), 230 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 899f7d3..9bb3ff7 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -123,57 +123,6 @@ static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = { }, }; -/* Function declarations */ -static int __init apex_init(void); -static void apex_exit(void); - -static int apex_add_dev_cb(struct gasket_dev *gasket_dev); - -static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev); - -static int apex_device_cleanup(struct gasket_dev *gasket_dev); - -static int apex_device_open_cb(struct gasket_dev *gasket_dev); - -static ssize_t sysfs_show( - struct device *device, struct device_attribute *attr, char *buf); - -static int apex_reset(struct gasket_dev *gasket_dev, uint type); - -static int apex_get_status(struct gasket_dev *gasket_dev); - -static bool apex_ioctl_check_permissions(struct file *file, uint cmd); - -static long apex_ioctl(struct file *file, uint cmd, void __user *argp); - -static long apex_clock_gating(struct gasket_dev *gasket_dev, - struct apex_gate_clock_ioctl __user *argp); - -static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulong arg); - -static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type); - -static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type); - -static bool is_gcb_in_reset(struct gasket_dev *gasket_dev); - -/* Data definitions */ - -/* The data necessary to display this file's sysfs entries. */ -static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { - GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show, - ATTR_KERNEL_HIB_PAGE_TABLE_SIZE), - GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show, - ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE), - GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, - ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), - GASKET_END_OF_ATTR_ARRAY -}; - -static const struct pci_device_id apex_pci_ids[] = { - { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 } -}; - /* The regions in the BAR2 space that can be mapped into user space. */ static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = { { 0x40000, 0x1000 }, @@ -253,65 +202,6 @@ static struct gasket_interrupt_desc apex_interrupts[] = { }, }; -static struct gasket_driver_desc apex_desc = { - .name = "apex", - .driver_version = APEX_DRIVER_VERSION, - .major = 120, - .minor = 0, - .module = THIS_MODULE, - .pci_id_table = apex_pci_ids, - - .num_page_tables = NUM_NODES, - .page_table_bar_index = APEX_BAR_INDEX, - .page_table_configs = apex_page_table_configs, - .page_table_extended_bit = APEX_EXTENDED_SHIFT, - - .bar_descriptions = { - GASKET_UNUSED_BAR, - GASKET_UNUSED_BAR, - { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET, - NUM_REGIONS, mappable_regions, PCI_BAR }, - GASKET_UNUSED_BAR, - GASKET_UNUSED_BAR, - GASKET_UNUSED_BAR, - }, - .coherent_buffer_description = { - APEX_CH_MEM_BYTES, - (VM_WRITE | VM_READ), - APEX_CM_OFFSET, - }, - .interrupt_type = PCI_MSIX, - .interrupt_bar_index = APEX_BAR_INDEX, - .num_interrupts = APEX_INTERRUPT_COUNT, - .interrupts = apex_interrupts, - .interrupt_pack_width = 7, - - .add_dev_cb = apex_add_dev_cb, - .remove_dev_cb = NULL, - - .enable_dev_cb = NULL, - .disable_dev_cb = NULL, - - .sysfs_setup_cb = apex_sysfs_setup_cb, - .sysfs_cleanup_cb = NULL, - - .device_open_cb = apex_device_open_cb, - .device_close_cb = apex_device_cleanup, - - .ioctl_handler_cb = apex_ioctl, - .device_status_cb = apex_get_status, - .hardware_revision_cb = NULL, - .device_reset_cb = apex_reset, -}; - -/* Module registration boilerplate */ -MODULE_DESCRIPTION("Google Apex driver"); -MODULE_VERSION(APEX_DRIVER_VERSION); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("John Joseph "); -MODULE_DEVICE_TABLE(pci, apex_pci_ids); -module_init(apex_init); -module_exit(apex_exit); /* Allows device to enter power save upon driver close(). */ static int allow_power_save = 1; @@ -331,61 +221,6 @@ module_param(allow_sw_clock_gating, int, 0644); module_param(allow_hw_clock_gating, int, 0644); module_param(bypass_top_level, int, 0644); -static int __init apex_init(void) -{ - return gasket_register_device(&apex_desc); -} - -static void apex_exit(void) -{ - gasket_unregister_device(&apex_desc); -} - -static int apex_add_dev_cb(struct gasket_dev *gasket_dev) -{ - ulong page_table_ready, msix_table_ready; - int retries = 0; - - apex_reset(gasket_dev, 0); - - while (retries < APEX_RESET_RETRY) { - page_table_ready = - gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); - msix_table_ready = - gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); - if (page_table_ready && msix_table_ready) - break; - schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); - retries++; - } - - if (retries == APEX_RESET_RETRY) { - if (!page_table_ready) - dev_err(gasket_dev->dev, "Page table init timed out\n"); - if (!msix_table_ready) - dev_err(gasket_dev->dev, "MSI-X table init timed out\n"); - return -ETIMEDOUT; - } - - return 0; -} - -static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) -{ - return gasket_sysfs_create_entries( - gasket_dev->dev_info.device, apex_sysfs_attrs); -} - -/* On device open, perform a core reinit reset. */ -static int apex_device_open_cb(struct gasket_dev *gasket_dev) -{ - return gasket_reset_nolock(gasket_dev, APEX_CHIP_REINIT_RESET); -} - /* Check the device status registers and return device status ALIVE or DEAD. */ static int apex_get_status(struct gasket_dev *gasket_dev) { @@ -393,53 +228,6 @@ static int apex_get_status(struct gasket_dev *gasket_dev) return GASKET_STATUS_ALIVE; } -/* Reset the Apex hardware. Called on final close via device_close_cb. */ -static int apex_device_cleanup(struct gasket_dev *gasket_dev) -{ - u64 scalar_error; - u64 hib_error; - int ret = 0; - - hib_error = gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_USER_HIB_ERROR_STATUS); - scalar_error = gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); - - dev_dbg(gasket_dev->dev, - "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n", - __func__, gasket_dev, hib_error, scalar_error); - - if (allow_power_save) - ret = apex_enter_reset(gasket_dev, APEX_CHIP_REINIT_RESET); - - return ret; -} - -/* Reset the hardware, then quit reset. Called on device open. */ -static int apex_reset(struct gasket_dev *gasket_dev, uint type) -{ - int ret; - - if (bypass_top_level) - return 0; - - if (!is_gcb_in_reset(gasket_dev)) { - /* We are not in reset - toggle the reset bit so as to force - * re-init of custom block - */ - dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__); - - ret = apex_enter_reset(gasket_dev, type); - if (ret) - return ret; - } - ret = apex_quit_reset(gasket_dev, type); - - return ret; -} - /* Enter GCB reset state. */ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) { @@ -578,6 +366,30 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) return 0; } +/* Reset the Apex hardware. Called on final close via device_close_cb. */ +static int apex_device_cleanup(struct gasket_dev *gasket_dev) +{ + u64 scalar_error; + u64 hib_error; + int ret = 0; + + hib_error = gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_ERROR_STATUS); + scalar_error = gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); + + dev_dbg(gasket_dev->dev, + "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n", + __func__, gasket_dev, hib_error, scalar_error); + + if (allow_power_save) + ret = apex_enter_reset(gasket_dev, APEX_CHIP_REINIT_RESET); + + return ret; +} + /* Determine if GCB is in reset state. */ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) { @@ -588,6 +400,62 @@ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) return (val & SCU3_CUR_RST_GCB_BIT_MASK); } +/* Reset the hardware, then quit reset. Called on device open. */ +static int apex_reset(struct gasket_dev *gasket_dev, uint type) +{ + int ret; + + if (bypass_top_level) + return 0; + + if (!is_gcb_in_reset(gasket_dev)) { + /* We are not in reset - toggle the reset bit so as to force + * re-init of custom block + */ + dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__); + + ret = apex_enter_reset(gasket_dev, type); + if (ret) + return ret; + } + ret = apex_quit_reset(gasket_dev, type); + + return ret; +} + +static int apex_add_dev_cb(struct gasket_dev *gasket_dev) +{ + ulong page_table_ready, msix_table_ready; + int retries = 0; + + apex_reset(gasket_dev, 0); + + while (retries < APEX_RESET_RETRY) { + page_table_ready = + gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); + msix_table_ready = + gasket_dev_read_64( + gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); + if (page_table_ready && msix_table_ready) + break; + schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); + retries++; + } + + if (retries == APEX_RESET_RETRY) { + if (!page_table_ready) + dev_err(gasket_dev->dev, "Page table init timed out\n"); + if (!msix_table_ready) + dev_err(gasket_dev->dev, "MSI-X table init timed out\n"); + return -ETIMEDOUT; + } + + return 0; +} + /* * Check permissions for Apex ioctls. * Returns true if the current user may execute this ioctl, and false otherwise. @@ -597,24 +465,6 @@ static bool apex_ioctl_check_permissions(struct file *filp, uint cmd) return !!(filp->f_mode & FMODE_WRITE); } -/* Apex-specific ioctl handler. */ -static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) -{ - struct gasket_dev *gasket_dev = filp->private_data; - - if (!apex_ioctl_check_permissions(filp, cmd)) - return -EPERM; - - switch (cmd) { - case APEX_IOCTL_GATE_CLOCK: - return apex_clock_gating(gasket_dev, argp); - case APEX_IOCTL_PERFORMANCE_EXPECTATION: - return apex_set_performance_expectation(gasket_dev, argp); - default: - return -ENOTTY; /* unknown command */ - } -} - /* Gates or un-gates Apex clock. */ static long apex_clock_gating(struct gasket_dev *gasket_dev, struct apex_gate_clock_ioctl __user *argp) @@ -711,6 +561,24 @@ static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulon return 0; } +/* Apex-specific ioctl handler. */ +static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) +{ + struct gasket_dev *gasket_dev = filp->private_data; + + if (!apex_ioctl_check_permissions(filp, cmd)) + return -EPERM; + + switch (cmd) { + case APEX_IOCTL_GATE_CLOCK: + return apex_clock_gating(gasket_dev, argp); + case APEX_IOCTL_PERFORMANCE_EXPECTATION: + return apex_set_performance_expectation(gasket_dev, argp); + default: + return -ENOTTY; /* unknown command */ + } +} + /* Display driver sysfs entries. */ static ssize_t sysfs_show( struct device *device, struct device_attribute *attr, char *buf) @@ -762,9 +630,103 @@ static ssize_t sysfs_show( return ret; } +static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { + GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show, + ATTR_KERNEL_HIB_PAGE_TABLE_SIZE), + GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show, + ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE), + GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, + ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), + GASKET_END_OF_ATTR_ARRAY +}; + +static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) +{ + return gasket_sysfs_create_entries( + gasket_dev->dev_info.device, apex_sysfs_attrs); +} + +/* On device open, perform a core reinit reset. */ +static int apex_device_open_cb(struct gasket_dev *gasket_dev) +{ + return gasket_reset_nolock(gasket_dev, APEX_CHIP_REINIT_RESET); +} + +static const struct pci_device_id apex_pci_ids[] = { + { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 } +}; + static void apex_pci_fixup_class(struct pci_dev *pdev) { pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class; } DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); + +static struct gasket_driver_desc apex_desc = { + .name = "apex", + .driver_version = APEX_DRIVER_VERSION, + .major = 120, + .minor = 0, + .module = THIS_MODULE, + .pci_id_table = apex_pci_ids, + + .num_page_tables = NUM_NODES, + .page_table_bar_index = APEX_BAR_INDEX, + .page_table_configs = apex_page_table_configs, + .page_table_extended_bit = APEX_EXTENDED_SHIFT, + + .bar_descriptions = { + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET, + NUM_REGIONS, mappable_regions, PCI_BAR }, + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + }, + .coherent_buffer_description = { + APEX_CH_MEM_BYTES, + (VM_WRITE | VM_READ), + APEX_CM_OFFSET, + }, + .interrupt_type = PCI_MSIX, + .interrupt_bar_index = APEX_BAR_INDEX, + .num_interrupts = APEX_INTERRUPT_COUNT, + .interrupts = apex_interrupts, + .interrupt_pack_width = 7, + + .add_dev_cb = apex_add_dev_cb, + .remove_dev_cb = NULL, + + .enable_dev_cb = NULL, + .disable_dev_cb = NULL, + + .sysfs_setup_cb = apex_sysfs_setup_cb, + .sysfs_cleanup_cb = NULL, + + .device_open_cb = apex_device_open_cb, + .device_close_cb = apex_device_cleanup, + + .ioctl_handler_cb = apex_ioctl, + .device_status_cb = apex_get_status, + .hardware_revision_cb = NULL, + .device_reset_cb = apex_reset, +}; + +static int __init apex_init(void) +{ + return gasket_register_device(&apex_desc); +} + +static void apex_exit(void) +{ + gasket_unregister_device(&apex_desc); +} +MODULE_DESCRIPTION("Google Apex driver"); +MODULE_VERSION(APEX_DRIVER_VERSION); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("John Joseph "); +MODULE_DEVICE_TABLE(pci, apex_pci_ids); +module_init(apex_init); +module_exit(apex_exit); From c538a3d634062255846e134f0aa88bb627977d6c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 29 Jul 2018 12:36:46 -0700 Subject: [PATCH 104/209] staging: gasket: apex: fix function param line continuation style Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); Many of these TODO items were previously cleaned up during the conversion to standard logging functions. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 119 ++++++++++++++++++++++++-------------------------- 1 file changed, 58 insertions(+), 61 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 9bb3ff7..7ef0cb4 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -240,9 +240,9 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) * - Software force GCB idle * - Enable GCB idle */ - gasket_read_modify_write_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER, 0x0, 1, 32); + gasket_read_modify_write_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER, + 0x0, 1, 32); /* - Initiate DMA pause */ gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX, @@ -259,16 +259,16 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) } /* - Enable GCB reset (0x1 to rg_rst_gcb) */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x1, 2, 2); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x1, 2, 2); /* - Enable GCB clock Gate (0x1 to rg_gated_gcb) */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x1, 2, 18); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x1, 2, 18); /* - Enable GCB memory shut down (0x3 to rg_force_ram_sd) */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x3, 2, 14); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 0x3, 2, 14); /* - Wait for RAM shutdown. */ if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, @@ -297,24 +297,24 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) * - b00: Not forced (HW controlled) * - b1x: Force disable */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x0, 2, 14); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 0x0, 2, 14); /* * - Disable software clock gate: * - b00: Not forced (HW controlled) * - b1x: Force disable */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x0, 2, 18); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x0, 2, 18); /* * - Disable GCB reset (rg_rst_gcb): * - b00: Not forced (HW controlled) * - b1x: Force disable = Force not Reset */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x2, 2, 2); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x2, 2, 2); /* - Wait for RAM enable. */ if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, @@ -338,27 +338,28 @@ static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) } if (!allow_hw_clock_gating) { - val0 = gasket_dev_read_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); /* Inactive and Sleep mode are disabled. */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x3, - SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, - SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); - val1 = gasket_dev_read_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + gasket_read_modify_write_32(gasket_dev, + APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 0x3, + SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, + SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); + val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); dev_dbg(gasket_dev->dev, "Disallow HW clock gating 0x%x -> 0x%x\n", val0, val1); } else { - val0 = gasket_dev_read_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); /* Inactive mode enabled - Sleep mode disabled. */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 2, - SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, - SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); - val1 = gasket_dev_read_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 2, + SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, + SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); + val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); dev_dbg(gasket_dev->dev, "Allow HW clock gating 0x%x -> 0x%x\n", val0, val1); } @@ -373,12 +374,10 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) u64 hib_error; int ret = 0; - hib_error = gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_USER_HIB_ERROR_STATUS); - scalar_error = gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); + hib_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_ERROR_STATUS); + scalar_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); dev_dbg(gasket_dev->dev, "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n", @@ -393,8 +392,8 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) /* Determine if GCB is in reset state. */ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) { - u32 val = gasket_dev_read_32( - gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3); + u32 val = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); /* Masks rg_rst_gcb bit of SCU_CTRL_2 */ return (val & SCU3_CUR_RST_GCB_BIT_MASK); @@ -432,13 +431,11 @@ static int apex_add_dev_cb(struct gasket_dev *gasket_dev) while (retries < APEX_RESET_RETRY) { page_table_ready = - gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); + gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); msix_table_ready = - gasket_dev_read_64( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); + gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); if (page_table_ready && msix_table_ready) break; schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); @@ -481,20 +478,20 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, if (ibuf.enable) { /* Quiesce AXI, gate GCB clock. */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, 16); - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, 2, 18); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, + 16); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, + 2, 18); } else { /* Un-gate GCB clock, un-quiesce AXI. */ - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, 2, 18); - gasket_read_modify_write_32( - gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, 16); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, + 2, 18); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, + 16); } return 0; } @@ -580,8 +577,8 @@ static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) } /* Display driver sysfs entries. */ -static ssize_t sysfs_show( - struct device *device, struct device_attribute *attr, char *buf) +static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, + char *buf) { int ret; struct gasket_dev *gasket_dev; @@ -642,8 +639,8 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) { - return gasket_sysfs_create_entries( - gasket_dev->dev_info.device, apex_sysfs_attrs); + return gasket_sysfs_create_entries(gasket_dev->dev_info.device, + apex_sysfs_attrs); } /* On device open, perform a core reinit reset. */ From 0c423ff06cd9a749011e840ab4eff7da28c4ccbf Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:34 -0700 Subject: [PATCH 105/209] staging: gasket: core: remove static function forward declarations Remove forward declarations of static functions, move code to avoid forward references, for kernel style. Change-Id: I24ee2667d72dd39e2eb0ac94c1c9790654d06b74 Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 1828 ++++++++++++++++++++++++------------------------- 1 file changed, 881 insertions(+), 947 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 3aa6a21..b5a7254 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -67,61 +67,6 @@ enum do_map_region_status { DO_MAP_REGION_INVALID, }; -/* Function declarations; comments are with definitions. */ -static int __init gasket_init(void); -static void __exit gasket_exit(void); - -static int gasket_pci_probe( - struct pci_dev *pci_dev, const struct pci_device_id *id); -static void gasket_pci_remove(struct pci_dev *pci_dev); - -static int gasket_setup_pci(struct pci_dev *pci_dev, struct gasket_dev *dev); -static void gasket_cleanup_pci(struct gasket_dev *dev); - -static int gasket_map_pci_bar(struct gasket_dev *dev, int bar_num); -static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num); - -static int gasket_alloc_dev( - struct gasket_internal_desc *internal_desc, struct device *dev, - struct gasket_dev **pdev, const char *kobj_name); -static void gasket_free_dev(struct gasket_dev *dev); - -static int gasket_find_dev_slot( - struct gasket_internal_desc *internal_desc, const char *kobj_name); - -static int gasket_add_cdev( - struct gasket_cdev_info *dev_info, - const struct file_operations *file_ops, struct module *owner); - -static int gasket_enable_dev( - struct gasket_internal_desc *internal_desc, - struct gasket_dev *gasket_dev); -static void gasket_disable_dev(struct gasket_dev *gasket_dev); - -static struct gasket_internal_desc *lookup_internal_desc( - struct pci_dev *pci_dev); - -static ssize_t gasket_sysfs_data_show( - struct device *device, struct device_attribute *attr, char *buf); - -static int gasket_mmap(struct file *filp, struct vm_area_struct *vma); -static int gasket_open(struct inode *inode, struct file *file); -static int gasket_release(struct inode *inode, struct file *file); -static long gasket_ioctl(struct file *filp, uint cmd, ulong arg); - -static int gasket_mm_vma_bar_offset( - const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, - ulong *bar_offset); -static bool gasket_mm_get_mapping_addrs( - const struct gasket_mappable_region *region, ulong bar_offset, - ulong requested_length, struct gasket_mappable_region *mappable_region, - ulong *virt_offset); -static enum do_map_region_status do_map_region( - const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, - struct gasket_mappable_region *map_region); - -static int gasket_get_hw_status(struct gasket_dev *gasket_dev); - /* Global data definitions. */ /* Mutex - only for framework-wide data. Other data should be protected by * finer-grained locks. @@ -157,48 +102,6 @@ enum gasket_sysfs_attribute_type { ATTR_USER_MEM_RANGES }; -/* File operations for all Gasket devices. */ -static const struct file_operations gasket_file_ops = { - .owner = THIS_MODULE, - .llseek = no_llseek, - .mmap = gasket_mmap, - .open = gasket_open, - .release = gasket_release, - .unlocked_ioctl = gasket_ioctl, -}; - -/* These attributes apply to all Gasket driver instances. */ -static const struct gasket_sysfs_attribute gasket_sysfs_generic_attrs[] = { - GASKET_SYSFS_RO(bar_offsets, gasket_sysfs_data_show, ATTR_BAR_OFFSETS), - GASKET_SYSFS_RO(bar_sizes, gasket_sysfs_data_show, ATTR_BAR_SIZES), - GASKET_SYSFS_RO(driver_version, gasket_sysfs_data_show, - ATTR_DRIVER_VERSION), - GASKET_SYSFS_RO(framework_version, gasket_sysfs_data_show, - ATTR_FRAMEWORK_VERSION), - GASKET_SYSFS_RO(device_type, gasket_sysfs_data_show, ATTR_DEVICE_TYPE), - GASKET_SYSFS_RO(revision, gasket_sysfs_data_show, - ATTR_HARDWARE_REVISION), - GASKET_SYSFS_RO(pci_address, gasket_sysfs_data_show, ATTR_PCI_ADDRESS), - GASKET_SYSFS_RO(status, gasket_sysfs_data_show, ATTR_STATUS), - GASKET_SYSFS_RO(is_device_owned, gasket_sysfs_data_show, - ATTR_IS_DEVICE_OWNED), - GASKET_SYSFS_RO(device_owner, gasket_sysfs_data_show, - ATTR_DEVICE_OWNER), - GASKET_SYSFS_RO(write_open_count, gasket_sysfs_data_show, - ATTR_WRITE_OPEN_COUNT), - GASKET_SYSFS_RO(reset_count, gasket_sysfs_data_show, ATTR_RESET_COUNT), - GASKET_SYSFS_RO(user_mem_ranges, gasket_sysfs_data_show, - ATTR_USER_MEM_RANGES), - GASKET_END_OF_ATTR_ARRAY -}; - -MODULE_DESCRIPTION("Google Gasket driver framework"); -MODULE_VERSION(GASKET_FRAMEWORK_VERSION); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Rob Springer "); -module_init(gasket_init); -module_exit(gasket_exit); - /* Perform a standard Gasket callback. */ static inline int check_and_invoke_callback( struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) @@ -239,165 +142,43 @@ static int gasket_owned_by_current_tgid(struct gasket_cdev_info *info) (info->ownership.owner == current->tgid)); } -static int __init gasket_init(void) +/* + * Find the next free gasket_internal_dev slot. + * + * Returns the located slot number on success or a negative number on failure. + */ +static int gasket_find_dev_slot( + struct gasket_internal_desc *internal_desc, const char *kobj_name) { int i; - pr_info("Performing one-time init of the Gasket framework.\n"); - /* Check for duplicates and find a free slot. */ - mutex_lock(&g_mutex); - for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { - g_descs[i].driver_desc = NULL; - mutex_init(&g_descs[i].mutex); - } + mutex_lock(&internal_desc->mutex); - gasket_sysfs_init(); - - mutex_unlock(&g_mutex); - return 0; -} - -static void __exit gasket_exit(void) -{ - /* No deinit/dealloc needed at present. */ - pr_info("Removing Gasket framework module.\n"); -} - -/* See gasket_core.h for description. */ -int gasket_register_device(const struct gasket_driver_desc *driver_desc) -{ - int i, ret; - int desc_idx = -1; - struct gasket_internal_desc *internal; - - pr_info("Initializing Gasket framework device\n"); - /* Check for duplicates and find a free slot. */ - mutex_lock(&g_mutex); - - for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { - if (g_descs[i].driver_desc == driver_desc) { - pr_err("%s driver already loaded/registered\n", - driver_desc->name); - mutex_unlock(&g_mutex); + /* Search for a previous instance of this device. */ + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (internal_desc->devs[i] && + strcmp(internal_desc->devs[i]->kobj_name, kobj_name) == 0) { + pr_err("Duplicate device %s\n", kobj_name); + mutex_unlock(&internal_desc->mutex); return -EBUSY; } } - /* This and the above loop could be combined, but this reads easier. */ - for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { - if (!g_descs[i].driver_desc) { - g_descs[i].driver_desc = driver_desc; - desc_idx = i; + /* Find a free device slot. */ + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (!internal_desc->devs[i]) break; - } } - mutex_unlock(&g_mutex); - pr_info("Loaded %s driver, framework version %s\n", - driver_desc->name, GASKET_FRAMEWORK_VERSION); - - if (desc_idx == -1) { - pr_err("Too many Gasket drivers loaded: %d\n", - GASKET_FRAMEWORK_DESC_MAX); + if (i == GASKET_DEV_MAX) { + pr_err("Too many registered devices; max %d\n", GASKET_DEV_MAX); + mutex_unlock(&internal_desc->mutex); return -EBUSY; } - /* Internal structure setup. */ - pr_debug("Performing initial internal structure setup.\n"); - internal = &g_descs[desc_idx]; - mutex_init(&internal->mutex); - memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX); - memset(&internal->pci, 0, sizeof(internal->pci)); - internal->pci.name = driver_desc->name; - internal->pci.id_table = driver_desc->pci_id_table; - internal->pci.probe = gasket_pci_probe; - internal->pci.remove = gasket_pci_remove; - internal->class = - class_create(driver_desc->module, driver_desc->name); - - if (IS_ERR(internal->class)) { - pr_err("Cannot register %s class [ret=%ld]\n", - driver_desc->name, PTR_ERR(internal->class)); - ret = PTR_ERR(internal->class); - goto unregister_gasket_driver; - } - - /* - * Not using pci_register_driver() (without underscores), as it - * depends on KBUILD_MODNAME, and this is a shared file. - */ - pr_debug("Registering PCI driver.\n"); - ret = __pci_register_driver( - &internal->pci, driver_desc->module, driver_desc->name); - if (ret) { - pr_err("cannot register pci driver [ret=%d]\n", ret); - goto fail1; - } - - pr_debug("Registering char driver.\n"); - ret = register_chrdev_region( - MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, - driver_desc->name); - if (ret) { - pr_err("cannot register char driver [ret=%d]\n", ret); - goto fail2; - } - - pr_info("Driver registered successfully.\n"); - return 0; - -fail2: - pci_unregister_driver(&internal->pci); - -fail1: - class_destroy(internal->class); - -unregister_gasket_driver: - mutex_lock(&g_mutex); - g_descs[desc_idx].driver_desc = NULL; - mutex_unlock(&g_mutex); - return ret; + mutex_unlock(&internal_desc->mutex); + return i; } -EXPORT_SYMBOL(gasket_register_device); - -/* See gasket_core.h for description. */ -void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) -{ - int i, desc_idx; - struct gasket_internal_desc *internal_desc = NULL; - - mutex_lock(&g_mutex); - for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { - if (g_descs[i].driver_desc == driver_desc) { - internal_desc = &g_descs[i]; - desc_idx = i; - break; - } - } - mutex_unlock(&g_mutex); - - if (!internal_desc) { - pr_err("request to unregister unknown desc: %s, %d:%d\n", - driver_desc->name, driver_desc->major, - driver_desc->minor); - return; - } - - unregister_chrdev_region( - MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX); - - pci_unregister_driver(&internal_desc->pci); - - class_destroy(internal_desc->class); - - /* Finally, effectively "remove" the driver. */ - mutex_lock(&g_mutex); - g_descs[desc_idx].driver_desc = NULL; - mutex_unlock(&g_mutex); - - pr_info("removed %s driver\n", driver_desc->name); -} -EXPORT_SYMBOL(gasket_unregister_device); /* * Allocate and initialize a Gasket device structure, add the device to the @@ -473,260 +254,6 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) kfree(gasket_dev); } -/* - * Find the next free gasket_internal_dev slot. - * - * Returns the located slot number on success or a negative number on failure. - */ -static int gasket_find_dev_slot( - struct gasket_internal_desc *internal_desc, const char *kobj_name) -{ - int i; - - mutex_lock(&internal_desc->mutex); - - /* Search for a previous instance of this device. */ - for (i = 0; i < GASKET_DEV_MAX; i++) { - if (internal_desc->devs[i] && - strcmp(internal_desc->devs[i]->kobj_name, kobj_name) == 0) { - pr_err("Duplicate device %s\n", kobj_name); - mutex_unlock(&internal_desc->mutex); - return -EBUSY; - } - } - - /* Find a free device slot. */ - for (i = 0; i < GASKET_DEV_MAX; i++) { - if (!internal_desc->devs[i]) - break; - } - - if (i == GASKET_DEV_MAX) { - pr_err("Too many registered devices; max %d\n", GASKET_DEV_MAX); - mutex_unlock(&internal_desc->mutex); - return -EBUSY; - } - - mutex_unlock(&internal_desc->mutex); - return i; -} - -/* - * PCI subsystem probe function. - * - * Called when a Gasket device is found. Allocates device metadata, maps device - * memory, and calls gasket_enable_dev to prepare the device for active use. - * - * Returns 0 if successful and a negative value otherwise. - */ -static int gasket_pci_probe( - struct pci_dev *pci_dev, const struct pci_device_id *id) -{ - int ret; - const char *kobj_name = dev_name(&pci_dev->dev); - struct gasket_internal_desc *internal_desc; - struct gasket_dev *gasket_dev; - const struct gasket_driver_desc *driver_desc; - struct device *parent; - - pr_info("Add Gasket device %s\n", kobj_name); - - mutex_lock(&g_mutex); - internal_desc = lookup_internal_desc(pci_dev); - mutex_unlock(&g_mutex); - if (!internal_desc) { - pr_err("PCI probe called for unknown driver type\n"); - return -ENODEV; - } - - driver_desc = internal_desc->driver_desc; - - parent = &pci_dev->dev; - ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); - if (ret) - return ret; - gasket_dev->pci_dev = pci_dev_get(pci_dev); - if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { - pr_err("Cannot create %s device %s [ret = %ld]\n", - driver_desc->name, gasket_dev->dev_info.name, - PTR_ERR(gasket_dev->dev_info.device)); - ret = -ENODEV; - goto fail1; - } - - ret = gasket_setup_pci(pci_dev, gasket_dev); - if (ret) - goto fail2; - - ret = check_and_invoke_callback(gasket_dev, driver_desc->add_dev_cb); - if (ret) { - dev_err(gasket_dev->dev, "Error in add device cb: %d\n", ret); - goto fail2; - } - - ret = gasket_sysfs_create_mapping( - gasket_dev->dev_info.device, gasket_dev); - if (ret) - goto fail3; - - /* - * Once we've created the mapping structures successfully, attempt to - * create a symlink to the pci directory of this object. - */ - ret = sysfs_create_link(&gasket_dev->dev_info.device->kobj, - &pci_dev->dev.kobj, dev_name(&pci_dev->dev)); - if (ret) { - dev_err(gasket_dev->dev, - "Cannot create sysfs pci link: %d\n", ret); - goto fail3; - } - ret = gasket_sysfs_create_entries( - gasket_dev->dev_info.device, gasket_sysfs_generic_attrs); - if (ret) - goto fail4; - - ret = check_and_invoke_callback( - gasket_dev, driver_desc->sysfs_setup_cb); - if (ret) { - dev_err(gasket_dev->dev, "Error in sysfs setup cb: %d\n", ret); - goto fail5; - } - - ret = gasket_enable_dev(internal_desc, gasket_dev); - if (ret) { - pr_err("cannot setup %s device\n", driver_desc->name); - gasket_disable_dev(gasket_dev); - goto fail5; - } - - if (driver_desc->device_close_cb) { - /* Perform a device cleanup, so as to place it in Power Save - * Mode. */ - ret = driver_desc->device_close_cb(gasket_dev); - if (ret) { - gasket_log_error( - gasket_dev, "Device cleanup cb returned %d.", ret); - } - } - - return 0; - -fail5: - check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); -fail4: -fail3: - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); -fail2: - gasket_cleanup_pci(gasket_dev); - check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); -fail1: - gasket_free_dev(gasket_dev); - return ret; -} - -/* - * PCI subsystem remove function. - * - * Called to remove a Gasket device. Finds the device in the device list and - * cleans up metadata. - */ -static void gasket_pci_remove(struct pci_dev *pci_dev) -{ - int i; - struct gasket_internal_desc *internal_desc; - struct gasket_dev *gasket_dev = NULL; - const struct gasket_driver_desc *driver_desc; - /* Find the device desc. */ - mutex_lock(&g_mutex); - internal_desc = lookup_internal_desc(pci_dev); - if (!internal_desc) { - mutex_unlock(&g_mutex); - return; - } - mutex_unlock(&g_mutex); - - driver_desc = internal_desc->driver_desc; - - /* Now find the specific device */ - mutex_lock(&internal_desc->mutex); - for (i = 0; i < GASKET_DEV_MAX; i++) { - if (internal_desc->devs[i] && - internal_desc->devs[i]->pci_dev == pci_dev) { - gasket_dev = internal_desc->devs[i]; - break; - } - } - mutex_unlock(&internal_desc->mutex); - - if (!gasket_dev) - return; - - pr_info("remove %s device %s\n", internal_desc->driver_desc->name, - gasket_dev->kobj_name); - - gasket_disable_dev(gasket_dev); - gasket_cleanup_pci(gasket_dev); - - check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); - - check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); - - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); - gasket_free_dev(gasket_dev); -} - -/* - * Setup PCI & set up memory mapping for the specified device. - * - * Enables the PCI device, reads the BAR registers and sets up pointers to the - * device's memory mapped IO space. - * - * Returns 0 on success and a negative value otherwise. - */ -static int gasket_setup_pci( - struct pci_dev *pci_dev, struct gasket_dev *gasket_dev) -{ - int i, mapped_bars, ret; - - ret = pci_enable_device(pci_dev); - if (ret) { - dev_err(gasket_dev->dev, "cannot enable PCI device\n"); - return ret; - } - - pci_set_master(pci_dev); - - for (i = 0; i < GASKET_NUM_BARS; i++) { - ret = gasket_map_pci_bar(gasket_dev, i); - if (ret) { - mapped_bars = i; - goto fail; - } - } - - return 0; - -fail: - for (i = 0; i < mapped_bars; i++) - gasket_unmap_pci_bar(gasket_dev, i); - - pci_disable_device(pci_dev); - return -ENOMEM; -} - -/* Unmaps memory and cleans up PCI for the specified device. */ -static void gasket_cleanup_pci(struct gasket_dev *gasket_dev) -{ - int i; - - for (i = 0; i < GASKET_NUM_BARS; i++) - gasket_unmap_pci_bar(gasket_dev, i); - - pci_disable_device(gasket_dev->pci_dev); -} - /* * Maps the specified bar into kernel space. * @@ -836,6 +363,264 @@ static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num) release_mem_region(base, bytes); } +/* + * Setup PCI & set up memory mapping for the specified device. + * + * Enables the PCI device, reads the BAR registers and sets up pointers to the + * device's memory mapped IO space. + * + * Returns 0 on success and a negative value otherwise. + */ +static int gasket_setup_pci( + struct pci_dev *pci_dev, struct gasket_dev *gasket_dev) +{ + int i, mapped_bars, ret; + + ret = pci_enable_device(pci_dev); + if (ret) { + dev_err(gasket_dev->dev, "cannot enable PCI device\n"); + return ret; + } + + pci_set_master(pci_dev); + + for (i = 0; i < GASKET_NUM_BARS; i++) { + ret = gasket_map_pci_bar(gasket_dev, i); + if (ret) { + mapped_bars = i; + goto fail; + } + } + + return 0; + +fail: + for (i = 0; i < mapped_bars; i++) + gasket_unmap_pci_bar(gasket_dev, i); + + pci_disable_device(pci_dev); + return -ENOMEM; +} + +/* Unmaps memory and cleans up PCI for the specified device. */ +static void gasket_cleanup_pci(struct gasket_dev *gasket_dev) +{ + int i; + + for (i = 0; i < GASKET_NUM_BARS; i++) + gasket_unmap_pci_bar(gasket_dev, i); + + pci_disable_device(gasket_dev->pci_dev); +} + +/* Determine the health of the Gasket device. */ +static int gasket_get_hw_status(struct gasket_dev *gasket_dev) +{ + int status; + int i; + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + + status = gasket_check_and_invoke_callback_nolock( + gasket_dev, driver_desc->device_status_cb); + if (status != GASKET_STATUS_ALIVE) { + dev_dbg(gasket_dev->dev, "Hardware reported status %d.\n", + status); + return status; + } + + status = gasket_interrupt_system_status(gasket_dev); + if (status != GASKET_STATUS_ALIVE) { + dev_dbg(gasket_dev->dev, + "Interrupt system reported status %d.\n", status); + return status; + } + + for (i = 0; i < driver_desc->num_page_tables; ++i) { + status = gasket_page_table_system_status( + gasket_dev->page_table[i]); + if (status != GASKET_STATUS_ALIVE) { + dev_dbg(gasket_dev->dev, + "Page table %d reported status %d.\n", + i, status); + return status; + } + } + + return GASKET_STATUS_ALIVE; +} + +static ssize_t gasket_write_mappable_regions( + char *buf, const struct gasket_driver_desc *driver_desc, int bar_index) +{ + int i; + ssize_t written; + ssize_t total_written = 0; + ulong min_addr, max_addr; + struct gasket_bar_desc bar_desc = + driver_desc->bar_descriptions[bar_index]; + + if (bar_desc.permissions == GASKET_NOMAP) + return 0; + for (i = 0; + i < bar_desc.num_mappable_regions && total_written < PAGE_SIZE; + i++) { + min_addr = bar_desc.mappable_regions[i].start - + driver_desc->legacy_mmap_address_offset; + max_addr = bar_desc.mappable_regions[i].start - + driver_desc->legacy_mmap_address_offset + + bar_desc.mappable_regions[i].length_bytes; + written = scnprintf(buf, PAGE_SIZE - total_written, + "0x%08lx-0x%08lx\n", min_addr, max_addr); + total_written += written; + buf += written; + } + return total_written; +} + +static ssize_t gasket_sysfs_data_show( + struct device *device, struct device_attribute *attr, char *buf) +{ + int i, ret = 0; + ssize_t current_written = 0; + const struct gasket_driver_desc *driver_desc; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + const struct gasket_bar_desc *bar_desc; + enum gasket_sysfs_attribute_type sysfs_type; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + dev_err(device, "No sysfs mapping found for device\n"); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + dev_err(device, "No sysfs attr found for device\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return 0; + } + + driver_desc = gasket_dev->internal_desc->driver_desc; + + sysfs_type = + (enum gasket_sysfs_attribute_type)gasket_attr->data.attr_type; + switch (sysfs_type) { + case ATTR_BAR_OFFSETS: + for (i = 0; i < GASKET_NUM_BARS; i++) { + bar_desc = &driver_desc->bar_descriptions[i]; + if (bar_desc->size == 0) + continue; + current_written = + snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i, + (ulong)bar_desc->base); + buf += current_written; + ret += current_written; + } + break; + case ATTR_BAR_SIZES: + for (i = 0; i < GASKET_NUM_BARS; i++) { + bar_desc = &driver_desc->bar_descriptions[i]; + if (bar_desc->size == 0) + continue; + current_written = + snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i, + (ulong)bar_desc->size); + buf += current_written; + ret += current_written; + } + break; + case ATTR_DRIVER_VERSION: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", + gasket_dev->internal_desc->driver_desc->driver_version); + break; + case ATTR_FRAMEWORK_VERSION: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", GASKET_FRAMEWORK_VERSION); + break; + case ATTR_DEVICE_TYPE: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", + gasket_dev->internal_desc->driver_desc->name); + break; + case ATTR_HARDWARE_REVISION: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", gasket_dev->hardware_revision); + break; + case ATTR_PCI_ADDRESS: + ret = snprintf(buf, PAGE_SIZE, "%s\n", gasket_dev->kobj_name); + break; + case ATTR_STATUS: + ret = snprintf( + buf, PAGE_SIZE, "%s\n", + gasket_num_name_lookup( + gasket_dev->status, gasket_status_name_table)); + break; + case ATTR_IS_DEVICE_OWNED: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.is_owned); + break; + case ATTR_DEVICE_OWNER: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.owner); + break; + case ATTR_WRITE_OPEN_COUNT: + ret = snprintf( + buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.write_open_count); + break; + case ATTR_RESET_COUNT: + ret = snprintf(buf, PAGE_SIZE, "%d\n", gasket_dev->reset_count); + break; + case ATTR_USER_MEM_RANGES: + for (i = 0; i < GASKET_NUM_BARS; ++i) { + current_written = gasket_write_mappable_regions( + buf, driver_desc, i); + buf += current_written; + ret += current_written; + } + break; + default: + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + +/* These attributes apply to all Gasket driver instances. */ +static const struct gasket_sysfs_attribute gasket_sysfs_generic_attrs[] = { + GASKET_SYSFS_RO(bar_offsets, gasket_sysfs_data_show, ATTR_BAR_OFFSETS), + GASKET_SYSFS_RO(bar_sizes, gasket_sysfs_data_show, ATTR_BAR_SIZES), + GASKET_SYSFS_RO(driver_version, gasket_sysfs_data_show, + ATTR_DRIVER_VERSION), + GASKET_SYSFS_RO(framework_version, gasket_sysfs_data_show, + ATTR_FRAMEWORK_VERSION), + GASKET_SYSFS_RO(device_type, gasket_sysfs_data_show, ATTR_DEVICE_TYPE), + GASKET_SYSFS_RO(revision, gasket_sysfs_data_show, + ATTR_HARDWARE_REVISION), + GASKET_SYSFS_RO(pci_address, gasket_sysfs_data_show, ATTR_PCI_ADDRESS), + GASKET_SYSFS_RO(status, gasket_sysfs_data_show, ATTR_STATUS), + GASKET_SYSFS_RO(is_device_owned, gasket_sysfs_data_show, + ATTR_IS_DEVICE_OWNED), + GASKET_SYSFS_RO(device_owner, gasket_sysfs_data_show, + ATTR_DEVICE_OWNER), + GASKET_SYSFS_RO(write_open_count, gasket_sysfs_data_show, + ATTR_WRITE_OPEN_COUNT), + GASKET_SYSFS_RO(reset_count, gasket_sysfs_data_show, ATTR_RESET_COUNT), + GASKET_SYSFS_RO(user_mem_ranges, gasket_sysfs_data_show, + ATTR_USER_MEM_RANGES), + GASKET_END_OF_ATTR_ARRAY +}; + /* Add a char device and related info. */ static int gasket_add_cdev( struct gasket_cdev_info *dev_info, @@ -856,84 +641,6 @@ static int gasket_add_cdev( return 0; } -/* Perform final init and marks the device as active. */ -static int gasket_enable_dev( - struct gasket_internal_desc *internal_desc, - struct gasket_dev *gasket_dev) -{ - int tbl_idx; - int ret; - const struct gasket_driver_desc *driver_desc = - internal_desc->driver_desc; - - ret = gasket_interrupt_init( - gasket_dev, driver_desc->name, - driver_desc->interrupt_type, driver_desc->interrupts, - driver_desc->num_interrupts, driver_desc->interrupt_pack_width, - driver_desc->interrupt_bar_index, - driver_desc->wire_interrupt_offsets); - if (ret) { - dev_err(gasket_dev->dev, - "Critical failure to allocate interrupts: %d\n", ret); - gasket_interrupt_cleanup(gasket_dev); - return ret; - } - - for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { - dev_dbg(gasket_dev->dev, "Initializing page table %d.\n", - tbl_idx); - ret = gasket_page_table_init( - &gasket_dev->page_table[tbl_idx], - &gasket_dev->bar_data[ - driver_desc->page_table_bar_index], - &driver_desc->page_table_configs[tbl_idx], - gasket_dev->dev, gasket_dev->pci_dev); - if (ret) { - dev_err(gasket_dev->dev, - "Couldn't init page table %d: %d\n", - tbl_idx, ret); - return ret; - } - /* - * Make sure that the page table is clear and set to simple - * addresses. - */ - gasket_page_table_reset(gasket_dev->page_table[tbl_idx]); - } - - /* - * hardware_revision_cb returns a positive integer (the rev) if - * successful.) - */ - ret = check_and_invoke_callback( - gasket_dev, driver_desc->hardware_revision_cb); - if (ret < 0) { - dev_err(gasket_dev->dev, - "Error getting hardware revision: %d\n", ret); - return ret; - } - gasket_dev->hardware_revision = ret; - - ret = check_and_invoke_callback(gasket_dev, driver_desc->enable_dev_cb); - if (ret) { - dev_err(gasket_dev->dev, "Error in enable device cb: %d\n", - ret); - return ret; - } - - /* device_status_cb returns a device status, not an error code. */ - gasket_dev->status = gasket_get_hw_status(gasket_dev); - if (gasket_dev->status == GASKET_STATUS_DEAD) - dev_err(gasket_dev->dev, "Device reported as unhealthy.\n"); - - ret = gasket_add_cdev( - &gasket_dev->dev_info, &gasket_file_ops, driver_desc->module); - if (ret) - return ret; - - return 0; -} - /* Disable device operations. */ static void gasket_disable_dev(struct gasket_dev *gasket_dev) { @@ -981,177 +688,6 @@ static struct gasket_internal_desc *lookup_internal_desc( return NULL; } -/** - * Lookup a name by number in a num_name table. - * @num: Number to lookup. - * @table: Array of num_name structures, the table for the lookup. - * - * Description: Searches for num in the table. If found, the - * corresponding name is returned; otherwise NULL - * is returned. - * - * The table must have a NULL name pointer at the end. - */ -const char *gasket_num_name_lookup( - uint num, const struct gasket_num_name *table) -{ - uint i = 0; - - while (table[i].snn_name) { - if (num == table[i].snn_num) - break; - ++i; - } - - return table[i].snn_name; -} -EXPORT_SYMBOL(gasket_num_name_lookup); - -/* - * Open the char device file. - * - * If the open is for writing, and the device is not owned, this process becomes - * the owner. If the open is for writing and the device is already owned by - * some other process, it is an error. If this process is the owner, increment - * the open count. - * - * Returns 0 if successful, a negative error number otherwise. - */ -static int gasket_open(struct inode *inode, struct file *filp) -{ - int ret; - struct gasket_dev *gasket_dev; - const struct gasket_driver_desc *driver_desc; - struct gasket_ownership *ownership; - char task_name[TASK_COMM_LEN]; - struct gasket_cdev_info *dev_info = - container_of(inode->i_cdev, struct gasket_cdev_info, cdev); - struct pid_namespace *pid_ns = task_active_pid_ns(current); - int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); - - gasket_dev = dev_info->gasket_dev_ptr; - driver_desc = gasket_dev->internal_desc->driver_desc; - ownership = &dev_info->ownership; - get_task_comm(task_name, current); - filp->private_data = gasket_dev; - inode->i_size = 0; - - dev_dbg(gasket_dev->dev, - "Attempting to open with tgid %u (%s) (f_mode: 0%03o, " - "fmode_write: %d is_root: %u)\n", - current->tgid, task_name, filp->f_mode, - (filp->f_mode & FMODE_WRITE), is_root); - - /* Always allow non-writing accesses. */ - if (!(filp->f_mode & FMODE_WRITE)) { - dev_dbg(gasket_dev->dev, "Allowing read-only opening.\n"); - return 0; - } - - mutex_lock(&gasket_dev->mutex); - - dev_dbg(gasket_dev->dev, - "Current owner open count (owning tgid %u): %d.\n", - ownership->owner, ownership->write_open_count); - - /* Opening a node owned by another TGID is an error (unless root) */ - if (ownership->is_owned && ownership->owner != current->tgid && - !is_root) { - dev_err(gasket_dev->dev, - "Process %u is opening a node held by %u.\n", - current->tgid, ownership->owner); - mutex_unlock(&gasket_dev->mutex); - return -EPERM; - } - - /* If the node is not owned, assign it to the current TGID. */ - if (!ownership->is_owned) { - ret = gasket_check_and_invoke_callback_nolock( - gasket_dev, driver_desc->device_open_cb); - if (ret) { - dev_err(gasket_dev->dev, - "Error in device open cb: %d\n", ret); - mutex_unlock(&gasket_dev->mutex); - return ret; - } - ownership->is_owned = 1; - ownership->owner = current->tgid; - dev_dbg(gasket_dev->dev, "Device owner is now tgid %u\n", - ownership->owner); - } - - ownership->write_open_count++; - - dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n", - ownership->owner, ownership->write_open_count); - - mutex_unlock(&gasket_dev->mutex); - return 0; -} - -/* - * Called on a close of the device file. If this process is the owner, - * decrement the open count. On last close by the owner, free up buffers and - * eventfd contexts, and release ownership. - * - * Returns 0 if successful, a negative error number otherwise. - */ -static int gasket_release(struct inode *inode, struct file *file) -{ - int i; - struct gasket_dev *gasket_dev; - struct gasket_ownership *ownership; - const struct gasket_driver_desc *driver_desc; - char task_name[TASK_COMM_LEN]; - struct gasket_cdev_info *dev_info = - container_of(inode->i_cdev, struct gasket_cdev_info, cdev); - struct pid_namespace *pid_ns = task_active_pid_ns(current); - int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); - - gasket_dev = dev_info->gasket_dev_ptr; - driver_desc = gasket_dev->internal_desc->driver_desc; - ownership = &dev_info->ownership; - get_task_comm(task_name, current); - mutex_lock(&gasket_dev->mutex); - - dev_dbg(gasket_dev->dev, - "Releasing device node. Call origin: tgid %u (%s) " - "(f_mode: 0%03o, fmode_write: %d, is_root: %u)\n", - current->tgid, task_name, file->f_mode, - (file->f_mode & FMODE_WRITE), is_root); - dev_dbg(gasket_dev->dev, "Current open count (owning tgid %u): %d\n", - ownership->owner, ownership->write_open_count); - - if (file->f_mode & FMODE_WRITE) { - ownership->write_open_count--; - if (ownership->write_open_count == 0) { - dev_dbg(gasket_dev->dev, "Device is now free\n"); - ownership->is_owned = 0; - ownership->owner = 0; - - /* Forces chip reset before we unmap the page tables. */ - driver_desc->device_reset_cb(gasket_dev, 0); - - for (i = 0; i < driver_desc->num_page_tables; ++i) { - gasket_page_table_unmap_all( - gasket_dev->page_table[i]); - gasket_page_table_garbage_collect( - gasket_dev->page_table[i]); - gasket_free_coherent_memory_all(gasket_dev, i); - } - - /* Closes device, enters power save. */ - gasket_check_and_invoke_callback_nolock( - gasket_dev, driver_desc->device_close_cb); - } - } - - dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n", - ownership->owner, ownership->write_open_count); - mutex_unlock(&gasket_dev->mutex); - return 0; -} - /* * Verifies that the user has permissions to perform the requested mapping and * that the provided descriptor/range is of adequate size to hold the range to @@ -1311,6 +847,36 @@ static bool gasket_mm_get_mapping_addrs( return false; } +/* + * Calculates the offset where the VMA range begins in its containing BAR. + * The offset is written into bar_offset on success. + * Returns zero on success, anything else on error. + */ +static int gasket_mm_vma_bar_offset( + const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, + ulong *bar_offset) +{ + ulong raw_offset; + int bar_index; + const struct gasket_driver_desc *driver_desc = + gasket_dev->internal_desc->driver_desc; + + raw_offset = (vma->vm_pgoff << PAGE_SHIFT) + + driver_desc->legacy_mmap_address_offset; + bar_index = gasket_get_bar_index(gasket_dev, raw_offset); + if (bar_index < 0) { + dev_err(gasket_dev->dev, + "Unable to find matching bar for address 0x%lx\n", + raw_offset); + trace_gasket_mmap_exit(bar_index); + return bar_index; + } + *bar_offset = + raw_offset - driver_desc->bar_descriptions[bar_index].base; + + return 0; +} + int gasket_mm_unmap_region( const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, const struct gasket_mappable_region *map_region) @@ -1417,36 +983,6 @@ fail: return DO_MAP_REGION_FAILURE; } -/* - * Calculates the offset where the VMA range begins in its containing BAR. - * The offset is written into bar_offset on success. - * Returns zero on success, anything else on error. - */ -static int gasket_mm_vma_bar_offset( - const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, - ulong *bar_offset) -{ - ulong raw_offset; - int bar_index; - const struct gasket_driver_desc *driver_desc = - gasket_dev->internal_desc->driver_desc; - - raw_offset = (vma->vm_pgoff << PAGE_SHIFT) + - driver_desc->legacy_mmap_address_offset; - bar_index = gasket_get_bar_index(gasket_dev, raw_offset); - if (bar_index < 0) { - dev_err(gasket_dev->dev, - "Unable to find matching bar for address 0x%lx\n", - raw_offset); - trace_gasket_mmap_exit(bar_index); - return bar_index; - } - *bar_offset = - raw_offset - driver_desc->bar_descriptions[bar_index].base; - - return 0; -} - /* Map a region of coherent memory. */ static int gasket_mmap_coherent( struct gasket_dev *gasket_dev, struct vm_area_struct *vma) @@ -1636,41 +1172,149 @@ fail: return ret; } -/* Determine the health of the Gasket device. */ -static int gasket_get_hw_status(struct gasket_dev *gasket_dev) +/* + * Open the char device file. + * + * If the open is for writing, and the device is not owned, this process becomes + * the owner. If the open is for writing and the device is already owned by + * some other process, it is an error. If this process is the owner, increment + * the open count. + * + * Returns 0 if successful, a negative error number otherwise. + */ +static int gasket_open(struct inode *inode, struct file *filp) +{ + int ret; + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc; + struct gasket_ownership *ownership; + char task_name[TASK_COMM_LEN]; + struct gasket_cdev_info *dev_info = + container_of(inode->i_cdev, struct gasket_cdev_info, cdev); + struct pid_namespace *pid_ns = task_active_pid_ns(current); + int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); + + gasket_dev = dev_info->gasket_dev_ptr; + driver_desc = gasket_dev->internal_desc->driver_desc; + ownership = &dev_info->ownership; + get_task_comm(task_name, current); + filp->private_data = gasket_dev; + inode->i_size = 0; + + dev_dbg(gasket_dev->dev, + "Attempting to open with tgid %u (%s) (f_mode: 0%03o, " + "fmode_write: %d is_root: %u)\n", + current->tgid, task_name, filp->f_mode, + (filp->f_mode & FMODE_WRITE), is_root); + + /* Always allow non-writing accesses. */ + if (!(filp->f_mode & FMODE_WRITE)) { + dev_dbg(gasket_dev->dev, "Allowing read-only opening.\n"); + return 0; + } + + mutex_lock(&gasket_dev->mutex); + + dev_dbg(gasket_dev->dev, + "Current owner open count (owning tgid %u): %d.\n", + ownership->owner, ownership->write_open_count); + + /* Opening a node owned by another TGID is an error (unless root) */ + if (ownership->is_owned && ownership->owner != current->tgid && + !is_root) { + dev_err(gasket_dev->dev, + "Process %u is opening a node held by %u.\n", + current->tgid, ownership->owner); + mutex_unlock(&gasket_dev->mutex); + return -EPERM; + } + + /* If the node is not owned, assign it to the current TGID. */ + if (!ownership->is_owned) { + ret = gasket_check_and_invoke_callback_nolock( + gasket_dev, driver_desc->device_open_cb); + if (ret) { + dev_err(gasket_dev->dev, + "Error in device open cb: %d\n", ret); + mutex_unlock(&gasket_dev->mutex); + return ret; + } + ownership->is_owned = 1; + ownership->owner = current->tgid; + dev_dbg(gasket_dev->dev, "Device owner is now tgid %u\n", + ownership->owner); + } + + ownership->write_open_count++; + + dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n", + ownership->owner, ownership->write_open_count); + + mutex_unlock(&gasket_dev->mutex); + return 0; +} + +/* + * Called on a close of the device file. If this process is the owner, + * decrement the open count. On last close by the owner, free up buffers and + * eventfd contexts, and release ownership. + * + * Returns 0 if successful, a negative error number otherwise. + */ +static int gasket_release(struct inode *inode, struct file *file) { - int status; int i; - const struct gasket_driver_desc *driver_desc = - gasket_dev->internal_desc->driver_desc; + struct gasket_dev *gasket_dev; + struct gasket_ownership *ownership; + const struct gasket_driver_desc *driver_desc; + char task_name[TASK_COMM_LEN]; + struct gasket_cdev_info *dev_info = + container_of(inode->i_cdev, struct gasket_cdev_info, cdev); + struct pid_namespace *pid_ns = task_active_pid_ns(current); + int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); - status = gasket_check_and_invoke_callback_nolock( - gasket_dev, driver_desc->device_status_cb); - if (status != GASKET_STATUS_ALIVE) { - dev_dbg(gasket_dev->dev, "Hardware reported status %d.\n", - status); - return status; - } + gasket_dev = dev_info->gasket_dev_ptr; + driver_desc = gasket_dev->internal_desc->driver_desc; + ownership = &dev_info->ownership; + get_task_comm(task_name, current); + mutex_lock(&gasket_dev->mutex); - status = gasket_interrupt_system_status(gasket_dev); - if (status != GASKET_STATUS_ALIVE) { - dev_dbg(gasket_dev->dev, - "Interrupt system reported status %d.\n", status); - return status; - } + dev_dbg(gasket_dev->dev, + "Releasing device node. Call origin: tgid %u (%s) " + "(f_mode: 0%03o, fmode_write: %d, is_root: %u)\n", + current->tgid, task_name, file->f_mode, + (file->f_mode & FMODE_WRITE), is_root); + dev_dbg(gasket_dev->dev, "Current open count (owning tgid %u): %d\n", + ownership->owner, ownership->write_open_count); - for (i = 0; i < driver_desc->num_page_tables; ++i) { - status = gasket_page_table_system_status( - gasket_dev->page_table[i]); - if (status != GASKET_STATUS_ALIVE) { - dev_dbg(gasket_dev->dev, - "Page table %d reported status %d.\n", - i, status); - return status; + if (file->f_mode & FMODE_WRITE) { + ownership->write_open_count--; + if (ownership->write_open_count == 0) { + dev_dbg(gasket_dev->dev, "Device is now free\n"); + ownership->is_owned = 0; + ownership->owner = 0; + + /* Forces chip reset before we unmap the page tables. */ + driver_desc->device_reset_cb(gasket_dev, 0); + + for (i = 0; i < driver_desc->num_page_tables; ++i) { + gasket_page_table_unmap_all( + gasket_dev->page_table[i]); + gasket_page_table_garbage_collect( + gasket_dev->page_table[i]); + gasket_free_coherent_memory_all(gasket_dev, i); + } + + /* Closes device, enters power save. */ + gasket_check_and_invoke_callback_nolock( + gasket_dev, driver_desc->device_close_cb); } } - return GASKET_STATUS_ALIVE; + dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n", + ownership->owner, ownership->write_open_count); + mutex_unlock(&gasket_dev->mutex); + return 0; } /* @@ -1712,6 +1356,276 @@ static long gasket_ioctl(struct file *filp, uint cmd, ulong arg) return gasket_handle_ioctl(filp, cmd, argp); } +/* File operations for all Gasket devices. */ +static const struct file_operations gasket_file_ops = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .mmap = gasket_mmap, + .open = gasket_open, + .release = gasket_release, + .unlocked_ioctl = gasket_ioctl, +}; + +/* Perform final init and marks the device as active. */ +static int gasket_enable_dev( + struct gasket_internal_desc *internal_desc, + struct gasket_dev *gasket_dev) +{ + int tbl_idx; + int ret; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + + ret = gasket_interrupt_init( + gasket_dev, driver_desc->name, + driver_desc->interrupt_type, driver_desc->interrupts, + driver_desc->num_interrupts, driver_desc->interrupt_pack_width, + driver_desc->interrupt_bar_index, + driver_desc->wire_interrupt_offsets); + if (ret) { + dev_err(gasket_dev->dev, + "Critical failure to allocate interrupts: %d\n", ret); + gasket_interrupt_cleanup(gasket_dev); + return ret; + } + + for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { + dev_dbg(gasket_dev->dev, "Initializing page table %d.\n", + tbl_idx); + ret = gasket_page_table_init( + &gasket_dev->page_table[tbl_idx], + &gasket_dev->bar_data[ + driver_desc->page_table_bar_index], + &driver_desc->page_table_configs[tbl_idx], + gasket_dev->dev, gasket_dev->pci_dev); + if (ret) { + dev_err(gasket_dev->dev, + "Couldn't init page table %d: %d\n", + tbl_idx, ret); + return ret; + } + /* + * Make sure that the page table is clear and set to simple + * addresses. + */ + gasket_page_table_reset(gasket_dev->page_table[tbl_idx]); + } + + /* + * hardware_revision_cb returns a positive integer (the rev) if + * successful.) + */ + ret = check_and_invoke_callback( + gasket_dev, driver_desc->hardware_revision_cb); + if (ret < 0) { + dev_err(gasket_dev->dev, + "Error getting hardware revision: %d\n", ret); + return ret; + } + gasket_dev->hardware_revision = ret; + + ret = check_and_invoke_callback(gasket_dev, driver_desc->enable_dev_cb); + if (ret) { + dev_err(gasket_dev->dev, "Error in enable device cb: %d\n", + ret); + return ret; + } + + /* device_status_cb returns a device status, not an error code. */ + gasket_dev->status = gasket_get_hw_status(gasket_dev); + if (gasket_dev->status == GASKET_STATUS_DEAD) + dev_err(gasket_dev->dev, "Device reported as unhealthy.\n"); + + ret = gasket_add_cdev( + &gasket_dev->dev_info, &gasket_file_ops, driver_desc->module); + if (ret) + return ret; + + return 0; +} + +/* + * PCI subsystem probe function. + * + * Called when a Gasket device is found. Allocates device metadata, maps device + * memory, and calls gasket_enable_dev to prepare the device for active use. + * + * Returns 0 if successful and a negative value otherwise. + */ +static int gasket_pci_probe( + struct pci_dev *pci_dev, const struct pci_device_id *id) +{ + int ret; + const char *kobj_name = dev_name(&pci_dev->dev); + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc; + struct device *parent; + + pr_info("Add Gasket device %s\n", kobj_name); + + mutex_lock(&g_mutex); + internal_desc = lookup_internal_desc(pci_dev); + mutex_unlock(&g_mutex); + if (!internal_desc) { + pr_err("PCI probe called for unknown driver type\n"); + return -ENODEV; + } + + driver_desc = internal_desc->driver_desc; + + parent = &pci_dev->dev; + ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); + if (ret) + return ret; + gasket_dev->pci_dev = pci_dev_get(pci_dev); + if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { + pr_err("Cannot create %s device %s [ret = %ld]\n", + driver_desc->name, gasket_dev->dev_info.name, + PTR_ERR(gasket_dev->dev_info.device)); + ret = -ENODEV; + goto fail1; + } + + ret = gasket_setup_pci(pci_dev, gasket_dev); + if (ret) + goto fail2; + + ret = check_and_invoke_callback(gasket_dev, driver_desc->add_dev_cb); + if (ret) { + dev_err(gasket_dev->dev, "Error in add device cb: %d\n", ret); + goto fail2; + } + + ret = gasket_sysfs_create_mapping( + gasket_dev->dev_info.device, gasket_dev); + if (ret) + goto fail3; + + /* + * Once we've created the mapping structures successfully, attempt to + * create a symlink to the pci directory of this object. + */ + ret = sysfs_create_link(&gasket_dev->dev_info.device->kobj, + &pci_dev->dev.kobj, dev_name(&pci_dev->dev)); + if (ret) { + dev_err(gasket_dev->dev, + "Cannot create sysfs pci link: %d\n", ret); + goto fail3; + } + ret = gasket_sysfs_create_entries( + gasket_dev->dev_info.device, gasket_sysfs_generic_attrs); + if (ret) + goto fail4; + + ret = check_and_invoke_callback( + gasket_dev, driver_desc->sysfs_setup_cb); + if (ret) { + dev_err(gasket_dev->dev, "Error in sysfs setup cb: %d\n", ret); + goto fail5; + } + + ret = gasket_enable_dev(internal_desc, gasket_dev); + if (ret) { + pr_err("cannot setup %s device\n", driver_desc->name); + gasket_disable_dev(gasket_dev); + goto fail5; + } + + return 0; + +fail5: + check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); +fail4: +fail3: + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); +fail2: + gasket_cleanup_pci(gasket_dev); + check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); +fail1: + gasket_free_dev(gasket_dev); + return ret; +} + +/* + * PCI subsystem remove function. + * + * Called to remove a Gasket device. Finds the device in the device list and + * cleans up metadata. + */ +static void gasket_pci_remove(struct pci_dev *pci_dev) +{ + int i; + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev = NULL; + const struct gasket_driver_desc *driver_desc; + /* Find the device desc. */ + mutex_lock(&g_mutex); + internal_desc = lookup_internal_desc(pci_dev); + if (!internal_desc) { + mutex_unlock(&g_mutex); + return; + } + mutex_unlock(&g_mutex); + + driver_desc = internal_desc->driver_desc; + + /* Now find the specific device */ + mutex_lock(&internal_desc->mutex); + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (internal_desc->devs[i] && + internal_desc->devs[i]->pci_dev == pci_dev) { + gasket_dev = internal_desc->devs[i]; + break; + } + } + mutex_unlock(&internal_desc->mutex); + + if (!gasket_dev) + return; + + pr_info("remove %s device %s\n", internal_desc->driver_desc->name, + gasket_dev->kobj_name); + + gasket_disable_dev(gasket_dev); + gasket_cleanup_pci(gasket_dev); + + check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); + + check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); + + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); + gasket_free_dev(gasket_dev); +} + +/** + * Lookup a name by number in a num_name table. + * @num: Number to lookup. + * @table: Array of num_name structures, the table for the lookup. + * + * Description: Searches for num in the table. If found, the + * corresponding name is returned; otherwise NULL + * is returned. + * + * The table must have a NULL name pointer at the end. + */ +const char *gasket_num_name_lookup( + uint num, const struct gasket_num_name *table) +{ + uint i = 0; + + while (table[i].snn_name) { + if (num == table[i].snn_num) + break; + ++i; + } + + return table[i].snn_name; +} +EXPORT_SYMBOL(gasket_num_name_lookup); + int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type) { int ret; @@ -1770,152 +1684,6 @@ gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( } EXPORT_SYMBOL(gasket_get_ioctl_permissions_cb); -static ssize_t gasket_write_mappable_regions( - char *buf, const struct gasket_driver_desc *driver_desc, int bar_index) -{ - int i; - ssize_t written; - ssize_t total_written = 0; - ulong min_addr, max_addr; - struct gasket_bar_desc bar_desc = - driver_desc->bar_descriptions[bar_index]; - - if (bar_desc.permissions == GASKET_NOMAP) - return 0; - for (i = 0; - i < bar_desc.num_mappable_regions && total_written < PAGE_SIZE; - i++) { - min_addr = bar_desc.mappable_regions[i].start - - driver_desc->legacy_mmap_address_offset; - max_addr = bar_desc.mappable_regions[i].start - - driver_desc->legacy_mmap_address_offset + - bar_desc.mappable_regions[i].length_bytes; - written = scnprintf(buf, PAGE_SIZE - total_written, - "0x%08lx-0x%08lx\n", min_addr, max_addr); - total_written += written; - buf += written; - } - return total_written; -} - -static ssize_t gasket_sysfs_data_show( - struct device *device, struct device_attribute *attr, char *buf) -{ - int i, ret = 0; - ssize_t current_written = 0; - const struct gasket_driver_desc *driver_desc; - struct gasket_dev *gasket_dev; - struct gasket_sysfs_attribute *gasket_attr; - const struct gasket_bar_desc *bar_desc; - enum gasket_sysfs_attribute_type sysfs_type; - - gasket_dev = gasket_sysfs_get_device_data(device); - if (!gasket_dev) { - dev_err(device, "No sysfs mapping found for device\n"); - return 0; - } - - gasket_attr = gasket_sysfs_get_attr(device, attr); - if (!gasket_attr) { - dev_err(device, "No sysfs attr found for device\n"); - gasket_sysfs_put_device_data(device, gasket_dev); - return 0; - } - - driver_desc = gasket_dev->internal_desc->driver_desc; - - sysfs_type = - (enum gasket_sysfs_attribute_type)gasket_attr->data.attr_type; - switch (sysfs_type) { - case ATTR_BAR_OFFSETS: - for (i = 0; i < GASKET_NUM_BARS; i++) { - bar_desc = &driver_desc->bar_descriptions[i]; - if (bar_desc->size == 0) - continue; - current_written = - snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i, - (ulong)bar_desc->base); - buf += current_written; - ret += current_written; - } - break; - case ATTR_BAR_SIZES: - for (i = 0; i < GASKET_NUM_BARS; i++) { - bar_desc = &driver_desc->bar_descriptions[i]; - if (bar_desc->size == 0) - continue; - current_written = - snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i, - (ulong)bar_desc->size); - buf += current_written; - ret += current_written; - } - break; - case ATTR_DRIVER_VERSION: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", - gasket_dev->internal_desc->driver_desc->driver_version); - break; - case ATTR_FRAMEWORK_VERSION: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", GASKET_FRAMEWORK_VERSION); - break; - case ATTR_DEVICE_TYPE: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", - gasket_dev->internal_desc->driver_desc->name); - break; - case ATTR_HARDWARE_REVISION: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", gasket_dev->hardware_revision); - break; - case ATTR_PCI_ADDRESS: - ret = snprintf(buf, PAGE_SIZE, "%s\n", gasket_dev->kobj_name); - break; - case ATTR_STATUS: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", - gasket_num_name_lookup( - gasket_dev->status, gasket_status_name_table)); - break; - case ATTR_IS_DEVICE_OWNED: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", - gasket_dev->dev_info.ownership.is_owned); - break; - case ATTR_DEVICE_OWNER: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", - gasket_dev->dev_info.ownership.owner); - break; - case ATTR_WRITE_OPEN_COUNT: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", - gasket_dev->dev_info.ownership.write_open_count); - break; - case ATTR_RESET_COUNT: - ret = snprintf(buf, PAGE_SIZE, "%d\n", gasket_dev->reset_count); - break; - case ATTR_USER_MEM_RANGES: - for (i = 0; i < GASKET_NUM_BARS; ++i) { - current_written = gasket_write_mappable_regions( - buf, driver_desc, i); - buf += current_written; - ret += current_written; - } - break; - default: - dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", - attr->attr.name); - ret = 0; - break; - } - - gasket_sysfs_put_attr(device, gasket_attr); - gasket_sysfs_put_device_data(device, gasket_dev); - return ret; -} - /* Get the driver structure for a given gasket_dev. * @dev: pointer to gasket_dev, implementing the requested driver. */ @@ -1964,3 +1732,169 @@ int gasket_wait_with_reschedule( return -ETIMEDOUT; } EXPORT_SYMBOL(gasket_wait_with_reschedule); + +/* See gasket_core.h for description. */ +int gasket_register_device(const struct gasket_driver_desc *driver_desc) +{ + int i, ret; + int desc_idx = -1; + struct gasket_internal_desc *internal; + + pr_info("Initializing Gasket framework device\n"); + /* Check for duplicates and find a free slot. */ + mutex_lock(&g_mutex); + + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc == driver_desc) { + pr_err("%s driver already loaded/registered\n", + driver_desc->name); + mutex_unlock(&g_mutex); + return -EBUSY; + } + } + + /* This and the above loop could be combined, but this reads easier. */ + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (!g_descs[i].driver_desc) { + g_descs[i].driver_desc = driver_desc; + desc_idx = i; + break; + } + } + mutex_unlock(&g_mutex); + + pr_info("Loaded %s driver, framework version %s\n", + driver_desc->name, GASKET_FRAMEWORK_VERSION); + + if (desc_idx == -1) { + pr_err("Too many Gasket drivers loaded: %d\n", + GASKET_FRAMEWORK_DESC_MAX); + return -EBUSY; + } + + /* Internal structure setup. */ + pr_debug("Performing initial internal structure setup.\n"); + internal = &g_descs[desc_idx]; + mutex_init(&internal->mutex); + memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX); + memset(&internal->pci, 0, sizeof(internal->pci)); + internal->pci.name = driver_desc->name; + internal->pci.id_table = driver_desc->pci_id_table; + internal->pci.probe = gasket_pci_probe; + internal->pci.remove = gasket_pci_remove; + internal->class = + class_create(driver_desc->module, driver_desc->name); + + if (IS_ERR(internal->class)) { + pr_err("Cannot register %s class [ret=%ld]\n", + driver_desc->name, PTR_ERR(internal->class)); + ret = PTR_ERR(internal->class); + goto unregister_gasket_driver; + } + + /* + * Not using pci_register_driver() (without underscores), as it + * depends on KBUILD_MODNAME, and this is a shared file. + */ + pr_debug("Registering PCI driver.\n"); + ret = __pci_register_driver( + &internal->pci, driver_desc->module, driver_desc->name); + if (ret) { + pr_err("cannot register pci driver [ret=%d]\n", ret); + goto fail1; + } + + pr_debug("Registering char driver.\n"); + ret = register_chrdev_region( + MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, + driver_desc->name); + if (ret) { + pr_err("cannot register char driver [ret=%d]\n", ret); + goto fail2; + } + + pr_info("Driver registered successfully.\n"); + return 0; + +fail2: + pci_unregister_driver(&internal->pci); + +fail1: + class_destroy(internal->class); + +unregister_gasket_driver: + mutex_lock(&g_mutex); + g_descs[desc_idx].driver_desc = NULL; + mutex_unlock(&g_mutex); + return ret; +} +EXPORT_SYMBOL(gasket_register_device); + +/* See gasket_core.h for description. */ +void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) +{ + int i, desc_idx; + struct gasket_internal_desc *internal_desc = NULL; + + mutex_lock(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc == driver_desc) { + internal_desc = &g_descs[i]; + desc_idx = i; + break; + } + } + mutex_unlock(&g_mutex); + + if (!internal_desc) { + pr_err("request to unregister unknown desc: %s, %d:%d\n", + driver_desc->name, driver_desc->major, + driver_desc->minor); + return; + } + + unregister_chrdev_region( + MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX); + + pci_unregister_driver(&internal_desc->pci); + + class_destroy(internal_desc->class); + + /* Finally, effectively "remove" the driver. */ + mutex_lock(&g_mutex); + g_descs[desc_idx].driver_desc = NULL; + mutex_unlock(&g_mutex); + + pr_info("removed %s driver\n", driver_desc->name); +} +EXPORT_SYMBOL(gasket_unregister_device); + +static int __init gasket_init(void) +{ + int i; + + pr_info("Performing one-time init of the Gasket framework.\n"); + /* Check for duplicates and find a free slot. */ + mutex_lock(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + g_descs[i].driver_desc = NULL; + mutex_init(&g_descs[i].mutex); + } + + gasket_sysfs_init(); + + mutex_unlock(&g_mutex); + return 0; +} + +static void __exit gasket_exit(void) +{ + /* No deinit/dealloc needed at present. */ + pr_info("Removing Gasket framework module.\n"); +} +MODULE_DESCRIPTION("Google Gasket driver framework"); +MODULE_VERSION(GASKET_FRAMEWORK_VERSION); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Rob Springer "); +module_init(gasket_init); +module_exit(gasket_exit); From ddd7eaeb8ccfe4116807cb5303c181c78fdb6c55 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:35 -0700 Subject: [PATCH 106/209] staging: gasket: ioctl: remove static function forward declarations Remove forward declarations of static functions, move code to avoid forward references, for kernel style. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 356 +++++++++++++++++++++++-------------------------- 1 file changed, 168 insertions(+), 188 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 55bdd7b..134d45a 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -23,194 +23,6 @@ #define trace_gasket_ioctl_config_coherent_allocator(x, ...) #endif -static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd); -static int gasket_set_event_fd(struct gasket_dev *dev, - struct gasket_interrupt_eventfd __user *argp); -static int gasket_read_page_table_size( - struct gasket_dev *gasket_dev, - struct gasket_page_table_ioctl __user *argp); -static int gasket_read_simple_page_table_size( - struct gasket_dev *gasket_dev, - struct gasket_page_table_ioctl __user *argp); -static int gasket_partition_page_table( - struct gasket_dev *gasket_dev, - struct gasket_page_table_ioctl __user *argp); -static int gasket_map_buffers(struct gasket_dev *gasket_dev, - struct gasket_page_table_ioctl __user *argp); -static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, - struct gasket_page_table_ioctl __user *argp); -static int gasket_config_coherent_allocator( - struct gasket_dev *gasket_dev, - struct gasket_coherent_alloc_config_ioctl __user *argp); - -/* - * standard ioctl dispatch function. - * @filp: File structure pointer describing this node usage session. - * @cmd: ioctl number to handle. - * @argp: ioctl-specific data pointer. - * - * Standard ioctl dispatcher; forwards operations to individual handlers. - */ -long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) -{ - struct gasket_dev *gasket_dev; - unsigned long arg = (unsigned long)argp; - gasket_ioctl_permissions_cb_t ioctl_permissions_cb; - int retval; - - gasket_dev = (struct gasket_dev *)filp->private_data; - trace_gasket_ioctl_entry(gasket_dev->dev_info.name, cmd); - - ioctl_permissions_cb = gasket_get_ioctl_permissions_cb(gasket_dev); - if (ioctl_permissions_cb) { - retval = ioctl_permissions_cb(filp, cmd, argp); - if (retval < 0) { - trace_gasket_ioctl_exit(retval); - return retval; - } else if (retval == 0) { - trace_gasket_ioctl_exit(-EPERM); - return -EPERM; - } - } else if (!gasket_ioctl_check_permissions(filp, cmd)) { - trace_gasket_ioctl_exit(-EPERM); - dev_dbg(gasket_dev->dev, "ioctl cmd=%x noperm\n", cmd); - return -EPERM; - } - - /* Tracing happens in this switch statement for all ioctls with - * an integer argrument, but ioctls with a struct argument - * that needs copying and decoding, that tracing is done within - * the handler call. - */ - switch (cmd) { - case GASKET_IOCTL_RESET: - trace_gasket_ioctl_integer_data(arg); - retval = gasket_reset(gasket_dev, arg); - break; - case GASKET_IOCTL_SET_EVENTFD: - retval = gasket_set_event_fd(gasket_dev, argp); - break; - case GASKET_IOCTL_CLEAR_EVENTFD: - trace_gasket_ioctl_integer_data(arg); - retval = gasket_interrupt_clear_eventfd( - gasket_dev->interrupt_data, (int)arg); - break; - case GASKET_IOCTL_PARTITION_PAGE_TABLE: - trace_gasket_ioctl_integer_data(arg); - retval = gasket_partition_page_table(gasket_dev, argp); - break; - case GASKET_IOCTL_NUMBER_PAGE_TABLES: - trace_gasket_ioctl_integer_data(gasket_dev->num_page_tables); - if (copy_to_user(argp, &gasket_dev->num_page_tables, - sizeof(uint64_t))) - retval = -EFAULT; - else - retval = 0; - break; - case GASKET_IOCTL_PAGE_TABLE_SIZE: - retval = gasket_read_page_table_size(gasket_dev, argp); - break; - case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: - retval = gasket_read_simple_page_table_size(gasket_dev, argp); - break; - case GASKET_IOCTL_MAP_BUFFER: - retval = gasket_map_buffers(gasket_dev, argp); - break; - case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: - retval = gasket_config_coherent_allocator(gasket_dev, argp); - break; - case GASKET_IOCTL_UNMAP_BUFFER: - retval = gasket_unmap_buffers(gasket_dev, argp); - break; - case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: - /* Clear interrupt counts doesn't take an arg, so use 0. */ - trace_gasket_ioctl_integer_data(0); - retval = gasket_interrupt_reset_counts(gasket_dev); - break; - default: - /* If we don't understand the ioctl, the best we can do is trace - * the arg. - */ - trace_gasket_ioctl_integer_data(arg); - dev_dbg(gasket_dev->dev, - "Unknown ioctl cmd=0x%x not caught by " - "gasket_is_supported_ioctl\n", - cmd); - retval = -EINVAL; - break; - } - - trace_gasket_ioctl_exit(retval); - return retval; -} - -/* - * Determines if an ioctl is part of the standard Gasket framework. - * @cmd: The ioctl number to handle. - * - * Returns 1 if the ioctl is supported and 0 otherwise. - */ -long gasket_is_supported_ioctl(uint cmd) -{ - switch (cmd) { - case GASKET_IOCTL_RESET: - case GASKET_IOCTL_SET_EVENTFD: - case GASKET_IOCTL_CLEAR_EVENTFD: - case GASKET_IOCTL_PARTITION_PAGE_TABLE: - case GASKET_IOCTL_NUMBER_PAGE_TABLES: - case GASKET_IOCTL_PAGE_TABLE_SIZE: - case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: - case GASKET_IOCTL_MAP_BUFFER: - case GASKET_IOCTL_UNMAP_BUFFER: - case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: - case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: - return 1; - default: - return 0; - } -} - -/* Check permissions for Gasket ioctls. */ -static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) -{ - bool alive; - bool read, write; - struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; - - alive = (gasket_dev->status == GASKET_STATUS_ALIVE); - if (!alive) - dev_dbg(gasket_dev->dev, "%s alive %d status %d\n", - __func__, alive, gasket_dev->status); - - read = !!(filp->f_mode & FMODE_READ); - write = !!(filp->f_mode & FMODE_WRITE); - - switch (cmd) { - case GASKET_IOCTL_RESET: - case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: - return write; - - case GASKET_IOCTL_PAGE_TABLE_SIZE: - case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: - case GASKET_IOCTL_NUMBER_PAGE_TABLES: - return read; - - case GASKET_IOCTL_PARTITION_PAGE_TABLE: - case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: - return alive && write; - - case GASKET_IOCTL_MAP_BUFFER: - case GASKET_IOCTL_UNMAP_BUFFER: - return alive && write; - - case GASKET_IOCTL_CLEAR_EVENTFD: - case GASKET_IOCTL_SET_EVENTFD: - return alive && write; - } - - return false; /* unknown permissions */ -} - /* Associate an eventfd with an interrupt. */ static int gasket_set_event_fd(struct gasket_dev *gasket_dev, struct gasket_interrupt_eventfd __user *argp) @@ -410,3 +222,171 @@ static int gasket_config_coherent_allocator( return 0; } + +/* Check permissions for Gasket ioctls. */ +static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) +{ + bool alive; + bool read, write; + struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data; + + alive = (gasket_dev->status == GASKET_STATUS_ALIVE); + if (!alive) + dev_dbg(gasket_dev->dev, "%s alive %d status %d\n", + __func__, alive, gasket_dev->status); + + read = !!(filp->f_mode & FMODE_READ); + write = !!(filp->f_mode & FMODE_WRITE); + + switch (cmd) { + case GASKET_IOCTL_RESET: + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + return write; + + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + return read; + + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: + return alive && write; + + case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_UNMAP_BUFFER: + return alive && write; + + case GASKET_IOCTL_CLEAR_EVENTFD: + case GASKET_IOCTL_SET_EVENTFD: + return alive && write; + } + + return false; /* unknown permissions */ +} + +/* + * standard ioctl dispatch function. + * @filp: File structure pointer describing this node usage session. + * @cmd: ioctl number to handle. + * @argp: ioctl-specific data pointer. + * + * Standard ioctl dispatcher; forwards operations to individual handlers. + */ +long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) +{ + struct gasket_dev *gasket_dev; + unsigned long arg = (unsigned long)argp; + gasket_ioctl_permissions_cb_t ioctl_permissions_cb; + int retval; + + gasket_dev = (struct gasket_dev *)filp->private_data; + trace_gasket_ioctl_entry(gasket_dev->dev_info.name, cmd); + + ioctl_permissions_cb = gasket_get_ioctl_permissions_cb(gasket_dev); + if (ioctl_permissions_cb) { + retval = ioctl_permissions_cb(filp, cmd, argp); + if (retval < 0) { + trace_gasket_ioctl_exit(retval); + return retval; + } else if (retval == 0) { + trace_gasket_ioctl_exit(-EPERM); + return -EPERM; + } + } else if (!gasket_ioctl_check_permissions(filp, cmd)) { + trace_gasket_ioctl_exit(-EPERM); + dev_dbg(gasket_dev->dev, "ioctl cmd=%x noperm\n", cmd); + return -EPERM; + } + + /* Tracing happens in this switch statement for all ioctls with + * an integer argrument, but ioctls with a struct argument + * that needs copying and decoding, that tracing is done within + * the handler call. + */ + switch (cmd) { + case GASKET_IOCTL_RESET: + trace_gasket_ioctl_integer_data(arg); + retval = gasket_reset(gasket_dev, arg); + break; + case GASKET_IOCTL_SET_EVENTFD: + retval = gasket_set_event_fd(gasket_dev, argp); + break; + case GASKET_IOCTL_CLEAR_EVENTFD: + trace_gasket_ioctl_integer_data(arg); + retval = gasket_interrupt_clear_eventfd( + gasket_dev->interrupt_data, (int)arg); + break; + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + trace_gasket_ioctl_integer_data(arg); + retval = gasket_partition_page_table(gasket_dev, argp); + break; + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + trace_gasket_ioctl_integer_data(gasket_dev->num_page_tables); + if (copy_to_user(argp, &gasket_dev->num_page_tables, + sizeof(uint64_t))) + retval = -EFAULT; + else + retval = 0; + break; + case GASKET_IOCTL_PAGE_TABLE_SIZE: + retval = gasket_read_page_table_size(gasket_dev, argp); + break; + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + retval = gasket_read_simple_page_table_size(gasket_dev, argp); + break; + case GASKET_IOCTL_MAP_BUFFER: + retval = gasket_map_buffers(gasket_dev, argp); + break; + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: + retval = gasket_config_coherent_allocator(gasket_dev, argp); + break; + case GASKET_IOCTL_UNMAP_BUFFER: + retval = gasket_unmap_buffers(gasket_dev, argp); + break; + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + /* Clear interrupt counts doesn't take an arg, so use 0. */ + trace_gasket_ioctl_integer_data(0); + retval = gasket_interrupt_reset_counts(gasket_dev); + break; + default: + /* If we don't understand the ioctl, the best we can do is trace + * the arg. + */ + trace_gasket_ioctl_integer_data(arg); + dev_dbg(gasket_dev->dev, + "Unknown ioctl cmd=0x%x not caught by " + "gasket_is_supported_ioctl\n", + cmd); + retval = -EINVAL; + break; + } + + trace_gasket_ioctl_exit(retval); + return retval; +} + +/* + * Determines if an ioctl is part of the standard Gasket framework. + * @cmd: The ioctl number to handle. + * + * Returns 1 if the ioctl is supported and 0 otherwise. + */ +long gasket_is_supported_ioctl(uint cmd) +{ + switch (cmd) { + case GASKET_IOCTL_RESET: + case GASKET_IOCTL_SET_EVENTFD: + case GASKET_IOCTL_CLEAR_EVENTFD: + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_UNMAP_BUFFER: + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: + return 1; + default: + return 0; + } +} From a30af6c49e7c512c7bcbc191250c1a9ae41077ae Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:36 -0700 Subject: [PATCH 107/209] staging: gasket: interrupt: remove static function forward declarations Remove forward declarations of static functions, move code to avoid forward references, for kernel style. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 499 ++++++++++++++++++++++----------------------- 1 file changed, 242 insertions(+), 257 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 27fde99..3079b59 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -70,32 +70,259 @@ struct gasket_interrupt_data { int irq; }; -/* Function definitions. */ -static ssize_t interrupt_sysfs_show( - struct device *device, struct device_attribute *attr, char *buf); - -static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id); - /* Structures to display interrupt counts in sysfs. */ enum interrupt_sysfs_attribute_type { ATTR_INTERRUPT_COUNTS, }; +/* Set up device registers for interrupt handling. */ +static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) +{ + int i; + int pack_shift; + ulong mask; + ulong value; + struct gasket_interrupt_data *interrupt_data = + gasket_dev->interrupt_data; + + if (!interrupt_data) { + dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n"); + return; + } + + dev_dbg(gasket_dev->dev, "Running interrupt setup\n"); + + if (interrupt_data->type == PLATFORM_WIRE || + interrupt_data->type == PCI_MSI) { + /* Nothing needs to be done for platform or PCI devices. */ + return; + } + + if (interrupt_data->type != PCI_MSIX) { + dev_dbg(gasket_dev->dev, + "Cannot handle unsupported interrupt type %d\n", + interrupt_data->type); + return; + } + + /* Setup the MSIX table. */ + + for (i = 0; i < interrupt_data->num_interrupts; i++) { + /* + * If the interrupt is not packed, we can write the index into + * the register directly. If not, we need to deal with a read- + * modify-write and shift based on the packing index. + */ + dev_dbg(gasket_dev->dev, + "Setting up interrupt index %d with index 0x%llx and " + "packing %d\n", + interrupt_data->interrupts[i].index, + interrupt_data->interrupts[i].reg, + interrupt_data->interrupts[i].packing); + if (interrupt_data->interrupts[i].packing == UNPACKED) { + value = interrupt_data->interrupts[i].index; + } else { + switch (interrupt_data->interrupts[i].packing) { + case PACK_0: + pack_shift = 0; + break; + case PACK_1: + pack_shift = interrupt_data->pack_width; + break; + case PACK_2: + pack_shift = 2 * interrupt_data->pack_width; + break; + case PACK_3: + pack_shift = 3 * interrupt_data->pack_width; + break; + default: + dev_dbg(gasket_dev->dev, + "Found interrupt description with " + "unknown enum %d\n", + interrupt_data->interrupts[i].packing); + return; + } + + mask = ~(0xFFFF << pack_shift); + value = gasket_dev_read_64( + gasket_dev, + interrupt_data->interrupt_bar_index, + interrupt_data->interrupts[i].reg) & + mask; + value |= interrupt_data->interrupts[i].index + << pack_shift; + } + gasket_dev_write_64(gasket_dev, value, + interrupt_data->interrupt_bar_index, + interrupt_data->interrupts[i].reg); + } +} + +static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) +{ + struct eventfd_ctx *ctx; + struct gasket_interrupt_data *interrupt_data = dev_id; + int interrupt = -1; + int i; + + /* If this linear lookup is a problem, we can maintain a map/hash. */ + for (i = 0; i < interrupt_data->num_interrupts; i++) { + if (interrupt_data->msix_entries[i].vector == irq) { + interrupt = interrupt_data->msix_entries[i].entry; + break; + } + } + if (interrupt == -1) { + pr_err("Received unknown irq %d\n", irq); + return IRQ_HANDLED; + } + trace_gasket_interrupt_event(interrupt_data->name, interrupt); + + ctx = interrupt_data->eventfd_ctxs[interrupt]; + if (ctx) + eventfd_signal(ctx, 1); + + ++(interrupt_data->interrupt_counts[interrupt]); + + return IRQ_HANDLED; +} + +static int gasket_interrupt_msix_init( + struct gasket_interrupt_data *interrupt_data) +{ + int ret = 1; + int i; + + for (i = 0; i < interrupt_data->num_interrupts; i++) { + interrupt_data->msix_entries[i].entry = i; + interrupt_data->msix_entries[i].vector = 0; + interrupt_data->eventfd_ctxs[i] = NULL; + } + + /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */ + for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++) + ret = pci_enable_msix_exact(interrupt_data->pci_dev, + interrupt_data->msix_entries, + interrupt_data->num_interrupts); + + if (ret) + return ret > 0 ? -EBUSY : ret; + interrupt_data->msix_configured = 1; + + for (i = 0; i < interrupt_data->num_interrupts; i++) { + ret = request_irq( + interrupt_data->msix_entries[i].vector, + gasket_msix_interrupt_handler, 0, interrupt_data->name, + interrupt_data); + + if (ret) { + dev_err(&interrupt_data->pci_dev->dev, + "Cannot get IRQ for interrupt %d, vector %d; " + "%d\n", + i, interrupt_data->msix_entries[i].vector, ret); + return ret; + } + + interrupt_data->num_configured++; + } + + return 0; +} + +/* + * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt + * setup code with MSIX vectors masked. This is wrong because nothing else in + * the driver will normally touch the MSIX vectors. + * + * As a temporary hack, force unmasking there. + * + * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after + * gasket_interrupt_msix_init(), and remove this code. + */ +static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev) +{ + int i; +#define MSIX_VECTOR_SIZE 16 +#define MSIX_MASK_BIT_OFFSET 12 +#define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800 + for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) { + /* Check if the MSIX vector is unmasked */ + ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE + + MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE; + u32 mask = + gasket_dev_read_32( + gasket_dev, + gasket_dev->interrupt_data->interrupt_bar_index, + location); + if (!(mask & 1)) + continue; + /* Unmask the msix vector (clear 32 bits) */ + gasket_dev_write_32( + gasket_dev, 0, + gasket_dev->interrupt_data->interrupt_bar_index, + location); + } +#undef MSIX_VECTOR_SIZE +#undef MSIX_MASK_BIT_OFFSET +#undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +} + +static ssize_t interrupt_sysfs_show( + struct device *device, struct device_attribute *attr, char *buf) +{ + int i, ret; + ssize_t written = 0, total_written = 0; + struct gasket_interrupt_data *interrupt_data; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + enum interrupt_sysfs_attribute_type sysfs_type; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + dev_dbg(device, "No sysfs mapping found for device\n"); + return 0; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + dev_dbg(device, "No sysfs attr data found for device\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return 0; + } + + sysfs_type = (enum interrupt_sysfs_attribute_type) + gasket_attr->data.attr_type; + interrupt_data = gasket_dev->interrupt_data; + switch (sysfs_type) { + case ATTR_INTERRUPT_COUNTS: + for (i = 0; i < interrupt_data->num_interrupts; ++i) { + written = + scnprintf(buf, PAGE_SIZE - total_written, + "0x%02x: %ld\n", i, + interrupt_data->interrupt_counts[i]); + total_written += written; + buf += written; + } + ret = total_written; + break; + default: + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = { GASKET_SYSFS_RO( interrupt_counts, interrupt_sysfs_show, ATTR_INTERRUPT_COUNTS), GASKET_END_OF_ATTR_ARRAY, }; -static void gasket_interrupt_setup(struct gasket_dev *gasket_dev); - -/* MSIX init and cleanup. */ -static int gasket_interrupt_msix_init( - struct gasket_interrupt_data *interrupt_data); -static void gasket_interrupt_msix_cleanup( - struct gasket_interrupt_data *interrupt_data); -static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev); - int gasket_interrupt_init( struct gasket_dev *gasket_dev, const char *name, int type, const struct gasket_interrupt_desc *interrupts, @@ -181,48 +408,6 @@ int gasket_interrupt_init( return 0; } -static int gasket_interrupt_msix_init( - struct gasket_interrupt_data *interrupt_data) -{ - int ret = 1; - int i; - - for (i = 0; i < interrupt_data->num_interrupts; i++) { - interrupt_data->msix_entries[i].entry = i; - interrupt_data->msix_entries[i].vector = 0; - interrupt_data->eventfd_ctxs[i] = NULL; - } - - /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */ - for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++) - ret = pci_enable_msix_exact(interrupt_data->pci_dev, - interrupt_data->msix_entries, - interrupt_data->num_interrupts); - - if (ret) - return ret > 0 ? -EBUSY : ret; - interrupt_data->msix_configured = 1; - - for (i = 0; i < interrupt_data->num_interrupts; i++) { - ret = request_irq( - interrupt_data->msix_entries[i].vector, - gasket_msix_interrupt_handler, 0, interrupt_data->name, - interrupt_data); - - if (ret) { - dev_err(&interrupt_data->pci_dev->dev, - "Cannot get IRQ for interrupt %d, vector %d; " - "%d\n", - i, interrupt_data->msix_entries[i].vector, ret); - return ret; - } - - interrupt_data->num_configured++; - } - - return 0; -} - static void gasket_interrupt_msix_cleanup( struct gasket_interrupt_data *interrupt_data) { @@ -238,44 +423,6 @@ static void gasket_interrupt_msix_cleanup( interrupt_data->msix_configured = 0; } -/* - * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt - * setup code with MSIX vectors masked. This is wrong because nothing else in - * the driver will normally touch the MSIX vectors. - * - * As a temporary hack, force unmasking there. - * - * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after - * gasket_interrupt_msix_init(), and remove this code. - */ -static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev) -{ - int i; -#define MSIX_VECTOR_SIZE 16 -#define MSIX_MASK_BIT_OFFSET 12 -#define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800 - for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) { - /* Check if the MSIX vector is unmasked */ - ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE + - MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE; - u32 mask = - gasket_dev_read_32( - gasket_dev, - gasket_dev->interrupt_data->interrupt_bar_index, - location); - if (!(mask & 1)) - continue; - /* Unmask the msix vector (clear 32 bits) */ - gasket_dev_write_32( - gasket_dev, 0, - gasket_dev->interrupt_data->interrupt_bar_index, - location); - } -#undef MSIX_VECTOR_SIZE -#undef MSIX_MASK_BIT_OFFSET -#undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE -} - int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) { int ret; @@ -327,89 +474,6 @@ int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev) return 0; } -/* Set up device registers for interrupt handling. */ -static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) -{ - int i; - int pack_shift; - ulong mask; - ulong value; - struct gasket_interrupt_data *interrupt_data = - gasket_dev->interrupt_data; - - if (!interrupt_data) { - dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n"); - return; - } - - dev_dbg(gasket_dev->dev, "Running interrupt setup\n"); - - if (interrupt_data->type == PLATFORM_WIRE || - interrupt_data->type == PCI_MSI) { - /* Nothing needs to be done for platform or PCI devices. */ - return; - } - - if (interrupt_data->type != PCI_MSIX) { - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); - return; - } - - /* Setup the MSIX table. */ - - for (i = 0; i < interrupt_data->num_interrupts; i++) { - /* - * If the interrupt is not packed, we can write the index into - * the register directly. If not, we need to deal with a read- - * modify-write and shift based on the packing index. - */ - dev_dbg(gasket_dev->dev, - "Setting up interrupt index %d with index 0x%llx and " - "packing %d\n", - interrupt_data->interrupts[i].index, - interrupt_data->interrupts[i].reg, - interrupt_data->interrupts[i].packing); - if (interrupt_data->interrupts[i].packing == UNPACKED) { - value = interrupt_data->interrupts[i].index; - } else { - switch (interrupt_data->interrupts[i].packing) { - case PACK_0: - pack_shift = 0; - break; - case PACK_1: - pack_shift = interrupt_data->pack_width; - break; - case PACK_2: - pack_shift = 2 * interrupt_data->pack_width; - break; - case PACK_3: - pack_shift = 3 * interrupt_data->pack_width; - break; - default: - dev_dbg(gasket_dev->dev, - "Found interrupt description with " - "unknown enum %d\n", - interrupt_data->interrupts[i].packing); - return; - } - - mask = ~(0xFFFF << pack_shift); - value = gasket_dev_read_64( - gasket_dev, - interrupt_data->interrupt_bar_index, - interrupt_data->interrupts[i].reg) & - mask; - value |= interrupt_data->interrupts[i].index - << pack_shift; - } - gasket_dev_write_64(gasket_dev, value, - interrupt_data->interrupt_bar_index, - interrupt_data->interrupts[i].reg); - } -} - /* See gasket_interrupt.h for description. */ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) { @@ -489,82 +553,3 @@ int gasket_interrupt_clear_eventfd( interrupt_data->eventfd_ctxs[interrupt] = NULL; return 0; } - -static ssize_t interrupt_sysfs_show( - struct device *device, struct device_attribute *attr, char *buf) -{ - int i, ret; - ssize_t written = 0, total_written = 0; - struct gasket_interrupt_data *interrupt_data; - struct gasket_dev *gasket_dev; - struct gasket_sysfs_attribute *gasket_attr; - enum interrupt_sysfs_attribute_type sysfs_type; - - gasket_dev = gasket_sysfs_get_device_data(device); - if (!gasket_dev) { - dev_dbg(device, "No sysfs mapping found for device\n"); - return 0; - } - - gasket_attr = gasket_sysfs_get_attr(device, attr); - if (!gasket_attr) { - dev_dbg(device, "No sysfs attr data found for device\n"); - gasket_sysfs_put_device_data(device, gasket_dev); - return 0; - } - - sysfs_type = (enum interrupt_sysfs_attribute_type) - gasket_attr->data.attr_type; - interrupt_data = gasket_dev->interrupt_data; - switch (sysfs_type) { - case ATTR_INTERRUPT_COUNTS: - for (i = 0; i < interrupt_data->num_interrupts; ++i) { - written = - scnprintf(buf, PAGE_SIZE - total_written, - "0x%02x: %ld\n", i, - interrupt_data->interrupt_counts[i]); - total_written += written; - buf += written; - } - ret = total_written; - break; - default: - dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", - attr->attr.name); - ret = 0; - break; - } - - gasket_sysfs_put_attr(device, gasket_attr); - gasket_sysfs_put_device_data(device, gasket_dev); - return ret; -} - -static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) -{ - struct eventfd_ctx *ctx; - struct gasket_interrupt_data *interrupt_data = dev_id; - int interrupt = -1; - int i; - - /* If this linear lookup is a problem, we can maintain a map/hash. */ - for (i = 0; i < interrupt_data->num_interrupts; i++) { - if (interrupt_data->msix_entries[i].vector == irq) { - interrupt = interrupt_data->msix_entries[i].entry; - break; - } - } - if (interrupt == -1) { - pr_err("Received unknown irq %d\n", irq); - return IRQ_HANDLED; - } - trace_gasket_interrupt_event(interrupt_data->name, interrupt); - - ctx = interrupt_data->eventfd_ctxs[interrupt]; - if (ctx) - eventfd_signal(ctx, 1); - - ++(interrupt_data->interrupt_counts[interrupt]); - - return IRQ_HANDLED; -} From 9aed73349f703d5e059160d0c14e47eed6eab24c Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:37 -0700 Subject: [PATCH 108/209] staging: gasket: pg tbl: remove static function forward declarations Remove forward declarations of static functions, move code to avoid forward references, for kernel style. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 1431 +++++++++++++++++++++---------------------- 1 file changed, 683 insertions(+), 748 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index b42f663..aa036b2 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -214,71 +214,6 @@ struct gasket_page_table { struct gasket_coherent_page_entry *coherent_pages; }; -/* Mapping declarations */ -static int gasket_map_simple_pages( - struct gasket_page_table *pg_tbl, ulong host_addr, - ulong dev_addr, uint num_pages); -static int gasket_map_extended_pages( - struct gasket_page_table *pg_tbl, ulong host_addr, - ulong dev_addr, uint num_pages); -static int gasket_perform_mapping( - struct gasket_page_table *pg_tbl, - struct gasket_page_table_entry *pte_base, u64 __iomem *att_base, - ulong host_addr, uint num_pages, int is_simple_mapping); - -static int gasket_alloc_simple_entries( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); -static int gasket_alloc_extended_entries( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries); -static int gasket_alloc_extended_subtable( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, - u64 __iomem *att_reg); - -/* Unmapping declarations */ -static void gasket_page_table_unmap_nolock( - struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages); -static void gasket_page_table_unmap_all_nolock( - struct gasket_page_table *pg_tbl); -static void gasket_unmap_simple_pages( - struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages); -static void gasket_unmap_extended_pages( - struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages); -static void gasket_perform_unmapping( - struct gasket_page_table *pg_tbl, - struct gasket_page_table_entry *pte_base, u64 __iomem *att_base, - uint num_pages, int is_simple_mapping); - -static void gasket_free_extended_subtable( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, - u64 __iomem *att_reg); -static bool gasket_release_page(struct page *page); - -/* Other/utility declarations */ -static inline bool gasket_addr_is_simple( - struct gasket_page_table *pg_tbl, ulong addr); -static bool gasket_is_simple_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); -static bool gasket_is_extended_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages); -static bool gasket_is_pte_range_free( - struct gasket_page_table_entry *pte, uint num_entries); -static void gasket_page_table_garbage_collect_nolock( - struct gasket_page_table *pg_tbl); - -/* Address format declarations */ -static ulong gasket_components_to_dev_address( - struct gasket_page_table *pg_tbl, int is_simple, uint page_index, - uint offset); -static int gasket_simple_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr); -static ulong gasket_extended_lvl0_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr); -static ulong gasket_extended_lvl1_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr); - -static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr); - -/* Public/exported functions */ /* See gasket_page_table.h for description. */ int gasket_page_table_init( struct gasket_page_table **ppg_tbl, @@ -353,6 +288,85 @@ int gasket_page_table_init( return 0; } +/* + * Check if a range of PTEs is free. + * The page table mutex must be held by the caller. + */ +static bool gasket_is_pte_range_free( + struct gasket_page_table_entry *ptes, uint num_entries) +{ + int i; + + for (i = 0; i < num_entries; i++) { + if (ptes[i].status != PTE_FREE) + return false; + } + + return true; +} + +/* + * Free a second level page [sub]table. + * The page table mutex must be held before this call. + */ +static void gasket_free_extended_subtable( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, + u64 __iomem *slot) +{ + /* Release the page table from the driver */ + pte->status = PTE_FREE; + + /* Release the page table from the device */ + writeq(0, slot); + /* Force sync around the address release. */ + mb(); + + if (pte->dma_addr) + dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + vfree(pte->sublevel); + + if (pte->page) + free_page((ulong)page_address(pte->page)); + + memset(pte, 0, sizeof(struct gasket_page_table_entry)); +} + +/* + * Actually perform collection. + * The page table mutex must be held by the caller. + */ +static void gasket_page_table_garbage_collect_nolock( + struct gasket_page_table *pg_tbl) +{ + struct gasket_page_table_entry *pte; + u64 __iomem *slot; + + /* XXX FIX ME XXX -- more efficient to keep a usage count */ + /* rather than scanning the second level page tables */ + + for (pte = pg_tbl->entries + pg_tbl->num_simple_entries, + slot = pg_tbl->base_slot + pg_tbl->num_simple_entries; + pte < pg_tbl->entries + pg_tbl->config.total_entries; + pte++, slot++) { + if (pte->status == PTE_INUSE) { + if (gasket_is_pte_range_free( + pte->sublevel, GASKET_PAGES_PER_SUBTABLE)) + gasket_free_extended_subtable( + pg_tbl, pte, slot); + } + } +} + +/* See gasket_page_table.h for description. */ +void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl) +{ + mutex_lock(&pg_tbl->mutex); + gasket_page_table_garbage_collect_nolock(pg_tbl); + mutex_unlock(&pg_tbl->mutex); +} + /* See gasket_page_table.h for description. */ void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl) { @@ -403,6 +417,610 @@ int gasket_page_table_partition( } EXPORT_SYMBOL(gasket_page_table_partition); +/* + * Return whether a host buffer was mapped as coherent memory. + * + * A Gasket page_table currently support one contiguous dma range, mapped to one + * contiguous virtual memory range. Check if the host_addr is within that range. + */ +static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) +{ + u64 min, max; + + /* whether the host address is within user virt range */ + if (!pg_tbl->coherent_pages) + return 0; + + min = (u64)pg_tbl->coherent_pages[0].user_virt; + max = min + PAGE_SIZE * pg_tbl->num_coherent_pages; + + return min <= host_addr && host_addr < max; +} + +/* + * Get and map last level page table buffers. + * + * slots is the location(s) to write device-mapped page address. If this is a + * simple mapping, these will be address translation registers. If this is + * an extended mapping, these will be within a second-level page table + * allocated by the host and so must have their __iomem attribute casted away. + */ +static int gasket_perform_mapping( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, + u64 __iomem *slots, ulong host_addr, uint num_pages, + int is_simple_mapping) +{ + int ret; + ulong offset; + struct page *page; + dma_addr_t dma_addr; + ulong page_addr; + int i; + + for (i = 0; i < num_pages; i++) { + page_addr = host_addr + i * PAGE_SIZE; + offset = page_addr & (PAGE_SIZE - 1); + dev_dbg(pg_tbl->device, "%s i %d\n", __func__, i); + if (is_coherent(pg_tbl, host_addr)) { + u64 off = + (u64)host_addr - + (u64)pg_tbl->coherent_pages[0].user_virt; + ptes[i].page = NULL; + ptes[i].offset = offset; + ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + + off + i * PAGE_SIZE; + } else { + ret = get_user_pages_fast( + page_addr - offset, 1, 1, &page); + + if (ret <= 0) { + dev_err(pg_tbl->device, + "get user pages failed for addr=0x%lx, " + "offset=0x%lx [ret=%d]\n", + page_addr, offset, ret); + return ret ? ret : -ENOMEM; + } + ++pg_tbl->num_active_pages; + + ptes[i].page = page; + ptes[i].offset = offset; + + /* Map the page into DMA space. */ + ptes[i].dma_addr = + dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + dev_dbg(pg_tbl->device, + "%s i %d pte %p pfn %p -> mapped %llx\n", + __func__, i, &ptes[i], + (void *)page_to_pfn(page), + (unsigned long long)ptes[i].dma_addr); + + if (ptes[i].dma_addr == -1) { + dev_dbg(pg_tbl->device, + "%s i %d -> fail to map page %llx " + "[pfn %p ohys %p]\n", + __func__, i, + (unsigned long long)ptes[i].dma_addr, + (void *)page_to_pfn(page), + (void *)page_to_phys(page)); + return -1; + } + /* Wait until the page is mapped. */ + mb(); + } + + /* Make the DMA-space address available to the device. */ + dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG; + + if (is_simple_mapping) { + writeq(dma_addr, &slots[i]); + } else { + ((u64 __force *)slots)[i] = dma_addr; + /* Extended page table vectors are in DRAM, + * and so need to be synced each time they are updated. + */ + dma_map_single(pg_tbl->device, + (void *)&((u64 __force *)slots)[i], + sizeof(u64), DMA_TO_DEVICE); + } + ptes[i].status = PTE_INUSE; + } + return 0; +} + +/* + * Return the index of the page for the address in the simple table. + * Does not perform validity checking. + */ +static int gasket_simple_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr) +{ + return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & + (pg_tbl->config.total_entries - 1); +} + +/* + * Return the level 0 page index for the given address. + * Does not perform validity checking. + */ +static ulong gasket_extended_lvl0_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr) +{ + return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & + ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); +} + +/* + * Return the level 1 page index for the given address. + * Does not perform validity checking. + */ +static ulong gasket_extended_lvl1_page_idx( + struct gasket_page_table *pg_tbl, ulong dev_addr) +{ + return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & + (GASKET_PAGES_PER_SUBTABLE - 1); +} + +/* + * Allocate page table entries in a simple table. + * The page table mutex must be held by the caller. + */ +static int gasket_alloc_simple_entries( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + if (!gasket_is_pte_range_free( + pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr), + num_pages)) + return -EBUSY; + + return 0; +} + +/* Safely return a page to the OS. */ +static bool gasket_release_page(struct page *page) +{ + if (!page) + return false; + + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); + + return true; +} + +/* + * Unmap and release mapped pages. + * The page table mutex must be held by the caller. + */ +static void gasket_perform_unmapping( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, + u64 __iomem *slots, uint num_pages, int is_simple_mapping) +{ + int i; + /* + * For each page table entry and corresponding entry in the device's + * address translation table: + */ + for (i = 0; i < num_pages; i++) { + /* release the address from the device, */ + if (is_simple_mapping || ptes[i].status == PTE_INUSE) + writeq(0, &slots[i]); + else + ((u64 __force *)slots)[i] = 0; + /* Force sync around the address release. */ + mb(); + + /* release the address from the driver, */ + if (ptes[i].status == PTE_INUSE) { + if (ptes[i].dma_addr) { + dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, + PAGE_SIZE, DMA_FROM_DEVICE); + } + if (gasket_release_page(ptes[i].page)) + --pg_tbl->num_active_pages; + } + ptes[i].status = PTE_FREE; + + /* and clear the PTE. */ + memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); + } +} + +/* + * Unmap and release pages mapped to simple addresses. + * The page table mutex must be held by the caller. + */ +static void gasket_unmap_simple_pages( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); + + gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot, + pg_tbl->base_slot + slot, num_pages, 1); +} + +/* + * Unmap and release buffers to extended addresses. + * The page table mutex must be held by the caller. + */ +static void gasket_unmap_extended_pages( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + uint slot_idx, remain, len; + struct gasket_page_table_entry *pte; + u64 __iomem *slot_base; + + remain = num_pages; + slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pg_tbl->entries + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + while (remain > 0) { + /* TODO: Add check to ensure pte remains valid? */ + len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); + + if (pte->status == PTE_INUSE) { + slot_base = (u64 __iomem *)(page_address(pte->page) + + pte->offset); + gasket_perform_unmapping( + pg_tbl, pte->sublevel + slot_idx, + slot_base + slot_idx, len, 0); + } + + remain -= len; + slot_idx = 0; + pte++; + } +} + +/* Evaluates to nonzero if the specified virtual address is simple. */ +static inline bool gasket_addr_is_simple( + struct gasket_page_table *pg_tbl, ulong addr) +{ + return !((addr) & (pg_tbl)->extended_flag); +} + +/* + * Convert (simple, page, offset) into a device address. + * Examples: + * Simple page 0, offset 32: + * Input (0, 0, 32), Output 0x20 + * Simple page 1000, offset 511: + * Input (0, 1000, 512), Output 0x3E81FF + * Extended page 0, offset 32: + * Input (0, 0, 32), Output 0x8000000020 + * Extended page 1000, offset 511: + * Input (1, 1000, 512), Output 0x8003E81FF + */ +static ulong gasket_components_to_dev_address( + struct gasket_page_table *pg_tbl, int is_simple, uint page_index, + uint offset) +{ + ulong lvl0_index, lvl1_index; + + if (is_simple) { + /* Return simple addresses directly. */ + lvl0_index = page_index & (pg_tbl->config.total_entries - 1); + return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; + } + + /* + * This could be compressed into fewer statements, but + * A) the compiler should optimize it + * B) this is not slow + * C) this is an uncommon operation + * D) this is actually readable this way. + */ + lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE; + lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1); + return (pg_tbl)->extended_flag | + (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) | + (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset; +} + +/* + * Validity checking for simple addresses. + * + * Verify that address translation commutes (from address to/from page + offset) + * and that the requested page range starts and ends within the set of + * currently-partitioned simple pages. + */ +static bool gasket_is_simple_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + ulong page_offset = dev_addr & (PAGE_SIZE - 1); + ulong page_index = + (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1); + + if (gasket_components_to_dev_address( + pg_tbl, 1, page_index, page_offset) != dev_addr) { + dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", + dev_addr); + return true; + } + + if (page_index >= pg_tbl->num_simple_entries) { + dev_err(pg_tbl->device, + "starting slot at %lu is too large, max is < %u\n", + page_index, pg_tbl->num_simple_entries); + return true; + } + + if (page_index + num_pages > pg_tbl->num_simple_entries) { + dev_err(pg_tbl->device, + "ending slot at %lu is too large, max is <= %u\n", + page_index + num_pages, pg_tbl->num_simple_entries); + return true; + } + + return false; +} + +/* + * Validity checking for extended addresses. + * + * Verify that address translation commutes (from address to/from page + + * offset) and that the requested page range starts and ends within the set of + * currently-partitioned extended pages. + */ +static bool gasket_is_extended_dev_addr_bad( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + /* Starting byte index of dev_addr into the first mapped page */ + ulong page_offset = dev_addr & (PAGE_SIZE - 1); + ulong page_global_idx, page_lvl0_idx; + ulong num_lvl0_pages; + ulong addr; + + /* check if the device address is out of bound */ + addr = dev_addr & ~((pg_tbl)->extended_flag); + if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { + dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n", + dev_addr); + return true; + } + + /* Find the starting sub-page index in the space of all sub-pages. */ + page_global_idx = (dev_addr / PAGE_SIZE) & + (pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1); + + /* Find the starting level 0 index. */ + page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + /* Get the count of affected level 0 pages. */ + num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) / + GASKET_PAGES_PER_SUBTABLE; + + if (gasket_components_to_dev_address( + pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { + dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", + dev_addr); + return true; + } + + if (page_lvl0_idx >= pg_tbl->num_extended_entries) { + dev_err(pg_tbl->device, + "starting level 0 slot at %lu is too large, max is < " + "%u\n", page_lvl0_idx, pg_tbl->num_extended_entries); + return true; + } + + if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) { + dev_err(pg_tbl->device, + "ending level 0 slot at %lu is too large, max is <= %u\n", + page_lvl0_idx + num_lvl0_pages, + pg_tbl->num_extended_entries); + return true; + } + + return false; +} + +/* + * Non-locking entry to unmapping routines. + * The page table mutex must be held by the caller. + */ +static void gasket_page_table_unmap_nolock( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +{ + if (!num_pages) + return; + + if (gasket_addr_is_simple(pg_tbl, dev_addr)) + gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages); + else + gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages); +} + +/* + * Allocate and map pages to simple addresses. + * If there is an error, no pages are mapped. + */ +static int gasket_map_simple_pages( + struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + uint num_pages) +{ + int ret; + uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); + + ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); + if (ret) { + dev_err(pg_tbl->device, + "page table slots %u (@ 0x%lx) to %u are not available\n", + slot_idx, dev_addr, slot_idx + num_pages - 1); + return ret; + } + + ret = gasket_perform_mapping( + pg_tbl, pg_tbl->entries + slot_idx, + pg_tbl->base_slot + slot_idx, host_addr, num_pages, 1); + + if (ret) { + gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); + dev_err(pg_tbl->device, "gasket_perform_mapping %d\n", ret); + } + return ret; +} + +/* + * Allocate a second level page table. + * The page table mutex must be held by the caller. + */ +static int gasket_alloc_extended_subtable( + struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, + u64 __iomem *slot) +{ + ulong page_addr, subtable_bytes; + dma_addr_t dma_addr; + + /* XXX FIX ME XXX this is inefficient for non-4K page sizes */ + + /* GFP_DMA flag must be passed to architectures for which + * part of the memory range is not considered DMA'able. + * This seems to be the case for Juno board with 4.5.0 Linaro kernel + */ + page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA); + if (!page_addr) + return -ENOMEM; + pte->page = virt_to_page((void *)page_addr); + pte->offset = 0; + + subtable_bytes = sizeof(struct gasket_page_table_entry) * + GASKET_PAGES_PER_SUBTABLE; + pte->sublevel = vzalloc(subtable_bytes); + if (!pte->sublevel) { + free_page(page_addr); + memset(pte, 0, sizeof(struct gasket_page_table_entry)); + return -ENOMEM; + } + + /* Map the page into DMA space. */ + pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + /* Wait until the page is mapped. */ + mb(); + + /* make the addresses available to the device */ + dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; + writeq(dma_addr, slot); + + pte->status = PTE_INUSE; + + return 0; +} + +/* + * Allocate slots in an extended page table. Check to see if a range of page + * table slots are available. If necessary, memory is allocated for second level + * page tables. + * + * Note that memory for second level page tables is allocated as needed, but + * that memory is only freed on the final close of the device file, when the + * page tables are repartitioned, or the the device is removed. If there is an + * error or if the full range of slots is not available, any memory + * allocated for second level page tables remains allocated until final close, + * repartition, or device removal. + * + * The page table mutex must be held by the caller. + */ +static int gasket_alloc_extended_entries( + struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries) +{ + int ret = 0; + uint remain, subtable_slot_idx, len; + struct gasket_page_table_entry *pte; + u64 __iomem *slot; + + remain = num_entries; + subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pg_tbl->entries + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + slot = pg_tbl->base_slot + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + while (remain > 0) { + len = min(remain, + GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx); + + if (pte->status == PTE_FREE) { + ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); + if (ret) { + dev_err(pg_tbl->device, + "no memory for extended addr subtable\n"); + return ret; + } + } else { + if (!gasket_is_pte_range_free( + pte->sublevel + subtable_slot_idx, len)) + return -EBUSY; + } + + remain -= len; + subtable_slot_idx = 0; + pte++; + slot++; + } + + return 0; +} + +/* + * gasket_map_extended_pages - Get and map buffers to extended addresses. + * If there is an error, no pages are mapped. + */ +static int gasket_map_extended_pages( + struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + uint num_pages) +{ + int ret; + ulong dev_addr_end; + uint slot_idx, remain, len; + struct gasket_page_table_entry *pte; + u64 __iomem *slot_base; + + ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages); + if (ret) { + dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; + dev_err(pg_tbl->device, + "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are " + "not available\n", + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), + dev_addr, + gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end), + gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end)); + return ret; + } + + remain = num_pages; + slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); + pte = pg_tbl->entries + pg_tbl->num_simple_entries + + gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); + + while (remain > 0) { + len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); + + slot_base = + (u64 __iomem *)(page_address(pte->page) + pte->offset); + ret = gasket_perform_mapping( + pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, + host_addr, len, 0); + if (ret) { + gasket_page_table_unmap_nolock( + pg_tbl, dev_addr, num_pages); + return ret; + } + + remain -= len; + slot_idx = 0; + pte++; + host_addr += len * PAGE_SIZE; + } + + return 0; +} + /* * See gasket_page_table.h for general description. * @@ -489,14 +1107,6 @@ void gasket_page_table_reset(struct gasket_page_table *pg_tbl) mutex_unlock(&pg_tbl->mutex); } -/* See gasket_page_table.h for description. */ -void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl) -{ - mutex_lock(&pg_tbl->mutex); - gasket_page_table_garbage_collect_nolock(pg_tbl); - mutex_unlock(&pg_tbl->mutex); -} - /* See gasket_page_table.h for description. */ int gasket_page_table_lookup_page( struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage, @@ -635,681 +1245,6 @@ int gasket_page_table_system_status(struct gasket_page_table *page_table) return GASKET_STATUS_ALIVE; } -/* - * Allocate and map pages to simple addresses. - * If there is an error, no pages are mapped. - */ -static int gasket_map_simple_pages( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) -{ - int ret; - uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); - - ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); - if (ret) { - dev_err(pg_tbl->device, - "page table slots %u (@ 0x%lx) to %u are not available\n", - slot_idx, dev_addr, slot_idx + num_pages - 1); - return ret; - } - - ret = gasket_perform_mapping( - pg_tbl, pg_tbl->entries + slot_idx, - pg_tbl->base_slot + slot_idx, host_addr, num_pages, 1); - - if (ret) { - gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); - dev_err(pg_tbl->device, "gasket_perform_mapping %d\n", ret); - } - return ret; -} - -/* - * gasket_map_extended_pages - Get and map buffers to extended addresses. - * If there is an error, no pages are mapped. - */ -static int gasket_map_extended_pages( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) -{ - int ret; - ulong dev_addr_end; - uint slot_idx, remain, len; - struct gasket_page_table_entry *pte; - u64 __iomem *slot_base; - - ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages); - if (ret) { - dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; - dev_err(pg_tbl->device, - "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are " - "not available\n", - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), - dev_addr, - gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end), - gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end)); - return ret; - } - - remain = num_pages; - slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pg_tbl->entries + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - while (remain > 0) { - len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); - - slot_base = - (u64 __iomem *)(page_address(pte->page) + pte->offset); - ret = gasket_perform_mapping( - pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, - host_addr, len, 0); - if (ret) { - gasket_page_table_unmap_nolock( - pg_tbl, dev_addr, num_pages); - return ret; - } - - remain -= len; - slot_idx = 0; - pte++; - host_addr += len * PAGE_SIZE; - } - - return 0; -} - -/* - * Get and map last level page table buffers. - * - * slots is the location(s) to write device-mapped page address. If this is a - * simple mapping, these will be address translation registers. If this is - * an extended mapping, these will be within a second-level page table - * allocated by the host and so must have their __iomem attribute casted away. - */ -static int gasket_perform_mapping( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, - u64 __iomem *slots, ulong host_addr, uint num_pages, - int is_simple_mapping) -{ - int ret; - ulong offset; - struct page *page; - dma_addr_t dma_addr; - ulong page_addr; - int i; - - for (i = 0; i < num_pages; i++) { - page_addr = host_addr + i * PAGE_SIZE; - offset = page_addr & (PAGE_SIZE - 1); - dev_dbg(pg_tbl->device, "%s i %d\n", __func__, i); - if (is_coherent(pg_tbl, host_addr)) { - u64 off = - (u64)host_addr - - (u64)pg_tbl->coherent_pages[0].user_virt; - ptes[i].page = NULL; - ptes[i].offset = offset; - ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + - off + i * PAGE_SIZE; - } else { - ret = get_user_pages_fast( - page_addr - offset, 1, 1, &page); - - if (ret <= 0) { - dev_err(pg_tbl->device, - "get user pages failed for addr=0x%lx, " - "offset=0x%lx [ret=%d]\n", - page_addr, offset, ret); - return ret ? ret : -ENOMEM; - } - ++pg_tbl->num_active_pages; - - ptes[i].page = page; - ptes[i].offset = offset; - - /* Map the page into DMA space. */ - ptes[i].dma_addr = - dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); - dev_dbg(pg_tbl->device, - "%s i %d pte %p pfn %p -> mapped %llx\n", - __func__, i, &ptes[i], - (void *)page_to_pfn(page), - (unsigned long long)ptes[i].dma_addr); - - if (ptes[i].dma_addr == -1) { - dev_dbg(pg_tbl->device, - "%s i %d -> fail to map page %llx " - "[pfn %p ohys %p]\n", - __func__, i, - (unsigned long long)ptes[i].dma_addr, - (void *)page_to_pfn(page), - (void *)page_to_phys(page)); - return -1; - } - /* Wait until the page is mapped. */ - mb(); - } - - /* Make the DMA-space address available to the device. */ - dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG; - - if (is_simple_mapping) { - writeq(dma_addr, &slots[i]); - } else { - ((u64 __force *)slots)[i] = dma_addr; - /* Extended page table vectors are in DRAM, - * and so need to be synced each time they are updated. - */ - dma_map_single(pg_tbl->device, - (void *)&((u64 __force *)slots)[i], - sizeof(u64), DMA_TO_DEVICE); - } - ptes[i].status = PTE_INUSE; - } - return 0; -} - -/* - * Allocate page table entries in a simple table. - * The page table mutex must be held by the caller. - */ -static int gasket_alloc_simple_entries( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) -{ - if (!gasket_is_pte_range_free( - pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr), - num_pages)) - return -EBUSY; - - return 0; -} - -/* - * Allocate slots in an extended page table. Check to see if a range of page - * table slots are available. If necessary, memory is allocated for second level - * page tables. - * - * Note that memory for second level page tables is allocated as needed, but - * that memory is only freed on the final close of the device file, when the - * page tables are repartitioned, or the the device is removed. If there is an - * error or if the full range of slots is not available, any memory - * allocated for second level page tables remains allocated until final close, - * repartition, or device removal. - * - * The page table mutex must be held by the caller. - */ -static int gasket_alloc_extended_entries( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries) -{ - int ret = 0; - uint remain, subtable_slot_idx, len; - struct gasket_page_table_entry *pte; - u64 __iomem *slot; - - remain = num_entries; - subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pg_tbl->entries + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - slot = pg_tbl->base_slot + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - while (remain > 0) { - len = min(remain, - GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx); - - if (pte->status == PTE_FREE) { - ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); - if (ret) { - dev_err(pg_tbl->device, - "no memory for extended addr subtable\n"); - return ret; - } - } else { - if (!gasket_is_pte_range_free( - pte->sublevel + subtable_slot_idx, len)) - return -EBUSY; - } - - remain -= len; - subtable_slot_idx = 0; - pte++; - slot++; - } - - return 0; -} - -/* - * Allocate a second level page table. - * The page table mutex must be held by the caller. - */ -static int gasket_alloc_extended_subtable( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, - u64 __iomem *slot) -{ - ulong page_addr, subtable_bytes; - dma_addr_t dma_addr; - - /* XXX FIX ME XXX this is inefficient for non-4K page sizes */ - - /* GFP_DMA flag must be passed to architectures for which - * part of the memory range is not considered DMA'able. - * This seems to be the case for Juno board with 4.5.0 Linaro kernel - */ - page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA); - if (!page_addr) - return -ENOMEM; - pte->page = virt_to_page((void *)page_addr); - pte->offset = 0; - - subtable_bytes = sizeof(struct gasket_page_table_entry) * - GASKET_PAGES_PER_SUBTABLE; - pte->sublevel = vzalloc(subtable_bytes); - if (!pte->sublevel) { - free_page(page_addr); - memset(pte, 0, sizeof(struct gasket_page_table_entry)); - return -ENOMEM; - } - - /* Map the page into DMA space. */ - pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); - /* Wait until the page is mapped. */ - mb(); - - /* make the addresses available to the device */ - dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; - writeq(dma_addr, slot); - - pte->status = PTE_INUSE; - - return 0; -} - -/* - * Non-locking entry to unmapping routines. - * The page table mutex must be held by the caller. - */ -static void gasket_page_table_unmap_nolock( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) -{ - if (!num_pages) - return; - - if (gasket_addr_is_simple(pg_tbl, dev_addr)) - gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages); - else - gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages); -} - -/* - * Unmap and release pages mapped to simple addresses. - * The page table mutex must be held by the caller. - */ -static void gasket_unmap_simple_pages( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) -{ - uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); - - gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot, - pg_tbl->base_slot + slot, num_pages, 1); -} - -/* - * Unmap and release buffers to extended addresses. - * The page table mutex must be held by the caller. - */ -static void gasket_unmap_extended_pages( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) -{ - uint slot_idx, remain, len; - struct gasket_page_table_entry *pte; - u64 __iomem *slot_base; - - remain = num_pages; - slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pg_tbl->entries + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - while (remain > 0) { - /* TODO: Add check to ensure pte remains valid? */ - len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); - - if (pte->status == PTE_INUSE) { - slot_base = (u64 __iomem *)(page_address(pte->page) + - pte->offset); - gasket_perform_unmapping( - pg_tbl, pte->sublevel + slot_idx, - slot_base + slot_idx, len, 0); - } - - remain -= len; - slot_idx = 0; - pte++; - } -} - -/* - * Unmap and release mapped pages. - * The page table mutex must be held by the caller. - */ -static void gasket_perform_unmapping( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, - u64 __iomem *slots, uint num_pages, int is_simple_mapping) -{ - int i; - /* - * For each page table entry and corresponding entry in the device's - * address translation table: - */ - for (i = 0; i < num_pages; i++) { - /* release the address from the device, */ - if (is_simple_mapping || ptes[i].status == PTE_INUSE) - writeq(0, &slots[i]); - else - ((u64 __force *)slots)[i] = 0; - /* Force sync around the address release. */ - mb(); - - /* release the address from the driver, */ - if (ptes[i].status == PTE_INUSE) { - if (ptes[i].dma_addr) { - dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, - PAGE_SIZE, DMA_FROM_DEVICE); - } - if (gasket_release_page(ptes[i].page)) - --pg_tbl->num_active_pages; - } - ptes[i].status = PTE_FREE; - - /* and clear the PTE. */ - memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); - } -} - -/* - * Free a second level page [sub]table. - * The page table mutex must be held before this call. - */ -static void gasket_free_extended_subtable( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, - u64 __iomem *slot) -{ - /* Release the page table from the driver */ - pte->status = PTE_FREE; - - /* Release the page table from the device */ - writeq(0, slot); - /* Force sync around the address release. */ - mb(); - - if (pte->dma_addr) - dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, - DMA_BIDIRECTIONAL); - - vfree(pte->sublevel); - - if (pte->page) - free_page((ulong)page_address(pte->page)); - - memset(pte, 0, sizeof(struct gasket_page_table_entry)); -} - -/* Safely return a page to the OS. */ -static bool gasket_release_page(struct page *page) -{ - if (!page) - return false; - - if (!PageReserved(page)) - SetPageDirty(page); - put_page(page); - - return true; -} - -/* Evaluates to nonzero if the specified virtual address is simple. */ -static inline bool gasket_addr_is_simple( - struct gasket_page_table *pg_tbl, ulong addr) -{ - return !((addr) & (pg_tbl)->extended_flag); -} - -/* - * Validity checking for simple addresses. - * - * Verify that address translation commutes (from address to/from page + offset) - * and that the requested page range starts and ends within the set of - * currently-partitioned simple pages. - */ -static bool gasket_is_simple_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) -{ - ulong page_offset = dev_addr & (PAGE_SIZE - 1); - ulong page_index = - (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1); - - if (gasket_components_to_dev_address( - pg_tbl, 1, page_index, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", - dev_addr); - return true; - } - - if (page_index >= pg_tbl->num_simple_entries) { - dev_err(pg_tbl->device, - "starting slot at %lu is too large, max is < %u\n", - page_index, pg_tbl->num_simple_entries); - return true; - } - - if (page_index + num_pages > pg_tbl->num_simple_entries) { - dev_err(pg_tbl->device, - "ending slot at %lu is too large, max is <= %u\n", - page_index + num_pages, pg_tbl->num_simple_entries); - return true; - } - - return false; -} - -/* - * Validity checking for extended addresses. - * - * Verify that address translation commutes (from address to/from page + - * offset) and that the requested page range starts and ends within the set of - * currently-partitioned extended pages. - */ -static bool gasket_is_extended_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) -{ - /* Starting byte index of dev_addr into the first mapped page */ - ulong page_offset = dev_addr & (PAGE_SIZE - 1); - ulong page_global_idx, page_lvl0_idx; - ulong num_lvl0_pages; - ulong addr; - - /* check if the device address is out of bound */ - addr = dev_addr & ~((pg_tbl)->extended_flag); - if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n", - dev_addr); - return true; - } - - /* Find the starting sub-page index in the space of all sub-pages. */ - page_global_idx = (dev_addr / PAGE_SIZE) & - (pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1); - - /* Find the starting level 0 index. */ - page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - /* Get the count of affected level 0 pages. */ - num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) / - GASKET_PAGES_PER_SUBTABLE; - - if (gasket_components_to_dev_address( - pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", - dev_addr); - return true; - } - - if (page_lvl0_idx >= pg_tbl->num_extended_entries) { - dev_err(pg_tbl->device, - "starting level 0 slot at %lu is too large, max is < " - "%u\n", page_lvl0_idx, pg_tbl->num_extended_entries); - return true; - } - - if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) { - dev_err(pg_tbl->device, - "ending level 0 slot at %lu is too large, max is <= %u\n", - page_lvl0_idx + num_lvl0_pages, - pg_tbl->num_extended_entries); - return true; - } - - return false; -} - -/* - * Check if a range of PTEs is free. - * The page table mutex must be held by the caller. - */ -static bool gasket_is_pte_range_free( - struct gasket_page_table_entry *ptes, uint num_entries) -{ - int i; - - for (i = 0; i < num_entries; i++) { - if (ptes[i].status != PTE_FREE) - return false; - } - - return true; -} - -/* - * Actually perform collection. - * The page table mutex must be held by the caller. - */ -static void gasket_page_table_garbage_collect_nolock( - struct gasket_page_table *pg_tbl) -{ - struct gasket_page_table_entry *pte; - u64 __iomem *slot; - - /* XXX FIX ME XXX -- more efficient to keep a usage count */ - /* rather than scanning the second level page tables */ - - for (pte = pg_tbl->entries + pg_tbl->num_simple_entries, - slot = pg_tbl->base_slot + pg_tbl->num_simple_entries; - pte < pg_tbl->entries + pg_tbl->config.total_entries; - pte++, slot++) { - if (pte->status == PTE_INUSE) { - if (gasket_is_pte_range_free( - pte->sublevel, GASKET_PAGES_PER_SUBTABLE)) - gasket_free_extended_subtable( - pg_tbl, pte, slot); - } - } -} - -/* - * Convert (simple, page, offset) into a device address. - * Examples: - * Simple page 0, offset 32: - * Input (0, 0, 32), Output 0x20 - * Simple page 1000, offset 511: - * Input (0, 1000, 512), Output 0x3E81FF - * Extended page 0, offset 32: - * Input (0, 0, 32), Output 0x8000000020 - * Extended page 1000, offset 511: - * Input (1, 1000, 512), Output 0x8003E81FF - */ -static ulong gasket_components_to_dev_address( - struct gasket_page_table *pg_tbl, int is_simple, uint page_index, - uint offset) -{ - ulong lvl0_index, lvl1_index; - - if (is_simple) { - /* Return simple addresses directly. */ - lvl0_index = page_index & (pg_tbl->config.total_entries - 1); - return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; - } - - /* - * This could be compressed into fewer statements, but - * A) the compiler should optimize it - * B) this is not slow - * C) this is an uncommon operation - * D) this is actually readable this way. - */ - lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE; - lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1); - return (pg_tbl)->extended_flag | - (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) | - (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset; -} - -/* - * Return the index of the page for the address in the simple table. - * Does not perform validity checking. - */ -static int gasket_simple_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr) -{ - return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & - (pg_tbl->config.total_entries - 1); -} - -/* - * Return the level 0 page index for the given address. - * Does not perform validity checking. - */ -static ulong gasket_extended_lvl0_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr) -{ - return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & - ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); -} - -/* - * Return the level 1 page index for the given address. - * Does not perform validity checking. - */ -static ulong gasket_extended_lvl1_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr) -{ - return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & - (GASKET_PAGES_PER_SUBTABLE - 1); -} - -/* - * Return whether a host buffer was mapped as coherent memory. - * - * A Gasket page_table currently support one contiguous dma range, mapped to one - * contiguous virtual memory range. Check if the host_addr is within that range. - */ -static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) -{ - u64 min, max; - - /* whether the host address is within user virt range */ - if (!pg_tbl->coherent_pages) - return 0; - - min = (u64)pg_tbl->coherent_pages[0].user_virt; - max = min + PAGE_SIZE * pg_tbl->num_coherent_pages; - - return min <= host_addr && host_addr < max; -} - /* Record the host_addr to coherent dma memory mapping. */ int gasket_set_user_virt( struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address, From cab8d68d1b57bff2954b0d4bd2d3e3e0e1e19922 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:38 -0700 Subject: [PATCH 109/209] staging: gasket: TODO: remove entry for static function declarations The static function declarations are removed, remove the TODO file entry for this. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index 7f4c13c..6aa2a7f 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,5 @@ This is a list of things that need to be done to get this driver out of the staging directory. -- Remove static function declarations. - Document sysfs files with Documentation/ABI/ entries. - Use misc interface instead of major number for driver version description. - Add descriptions of module_param's From 7754d4728bf07d46ef4f65f21bb95363da087479 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:39 -0700 Subject: [PATCH 110/209] staging: gasket: core: fix function param line continuation style Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 294 +++++++++++++++++++++++++------------------------- gasket_core.h | 68 ++++++------ 2 files changed, 179 insertions(+), 183 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index b5a7254..4434452 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -103,8 +103,9 @@ enum gasket_sysfs_attribute_type { }; /* Perform a standard Gasket callback. */ -static inline int check_and_invoke_callback( - struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) +static inline int +check_and_invoke_callback(struct gasket_dev *gasket_dev, + int (*cb_function)(struct gasket_dev *)) { int ret = 0; @@ -119,8 +120,9 @@ static inline int check_and_invoke_callback( } /* Perform a standard Gasket callback without grabbing gasket_dev->mutex. */ -static inline int gasket_check_and_invoke_callback_nolock( - struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *)) +static inline int +gasket_check_and_invoke_callback_nolock(struct gasket_dev *gasket_dev, + int (*cb_function)(struct gasket_dev *)) { int ret = 0; @@ -147,8 +149,8 @@ static int gasket_owned_by_current_tgid(struct gasket_cdev_info *info) * * Returns the located slot number on success or a negative number on failure. */ -static int gasket_find_dev_slot( - struct gasket_internal_desc *internal_desc, const char *kobj_name) +static int gasket_find_dev_slot(struct gasket_internal_desc *internal_desc, + const char *kobj_name) { int i; @@ -186,9 +188,9 @@ static int gasket_find_dev_slot( * * Returns 0 if successful, a negative error code otherwise. */ -static int gasket_alloc_dev( - struct gasket_internal_desc *internal_desc, struct device *parent, - struct gasket_dev **pdev, const char *kobj_name) +static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, + struct device *parent, struct gasket_dev **pdev, + const char *kobj_name) { int dev_idx; const struct gasket_driver_desc *driver_desc = @@ -228,7 +230,7 @@ static int gasket_alloc_dev( gasket_dev->dev_idx); dev_info->devt = MKDEV(driver_desc->major, driver_desc->minor + - gasket_dev->dev_idx); + gasket_dev->dev_idx); dev_info->device = device_create(internal_desc->class, parent, dev_info->devt, gasket_dev, dev_info->name); @@ -371,8 +373,8 @@ static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num) * * Returns 0 on success and a negative value otherwise. */ -static int gasket_setup_pci( - struct pci_dev *pci_dev, struct gasket_dev *gasket_dev) +static int gasket_setup_pci(struct pci_dev *pci_dev, + struct gasket_dev *gasket_dev) { int i, mapped_bars, ret; @@ -421,8 +423,8 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) const struct gasket_driver_desc *driver_desc = gasket_dev->internal_desc->driver_desc; - status = gasket_check_and_invoke_callback_nolock( - gasket_dev, driver_desc->device_status_cb); + status = gasket_check_and_invoke_callback_nolock(gasket_dev, + driver_desc->device_status_cb); if (status != GASKET_STATUS_ALIVE) { dev_dbg(gasket_dev->dev, "Hardware reported status %d.\n", status); @@ -437,8 +439,7 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) } for (i = 0; i < driver_desc->num_page_tables; ++i) { - status = gasket_page_table_system_status( - gasket_dev->page_table[i]); + status = gasket_page_table_system_status(gasket_dev->page_table[i]); if (status != GASKET_STATUS_ALIVE) { dev_dbg(gasket_dev->dev, "Page table %d reported status %d.\n", @@ -450,8 +451,10 @@ static int gasket_get_hw_status(struct gasket_dev *gasket_dev) return GASKET_STATUS_ALIVE; } -static ssize_t gasket_write_mappable_regions( - char *buf, const struct gasket_driver_desc *driver_desc, int bar_index) +static ssize_t +gasket_write_mappable_regions(char *buf, + const struct gasket_driver_desc *driver_desc, + int bar_index) { int i; ssize_t written; @@ -478,8 +481,8 @@ static ssize_t gasket_write_mappable_regions( return total_written; } -static ssize_t gasket_sysfs_data_show( - struct device *device, struct device_attribute *attr, char *buf) +static ssize_t gasket_sysfs_data_show(struct device *device, + struct device_attribute *attr, char *buf) { int i, ret = 0; ssize_t current_written = 0; @@ -532,54 +535,49 @@ static ssize_t gasket_sysfs_data_show( } break; case ATTR_DRIVER_VERSION: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", - gasket_dev->internal_desc->driver_desc->driver_version); + ret = snprintf(buf, PAGE_SIZE, "%s\n", + gasket_dev->internal_desc->driver_desc->driver_version); break; case ATTR_FRAMEWORK_VERSION: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", GASKET_FRAMEWORK_VERSION); + ret = snprintf(buf, PAGE_SIZE, "%s\n", + GASKET_FRAMEWORK_VERSION); break; case ATTR_DEVICE_TYPE: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", - gasket_dev->internal_desc->driver_desc->name); + ret = snprintf(buf, PAGE_SIZE, "%s\n", + gasket_dev->internal_desc->driver_desc->name); break; case ATTR_HARDWARE_REVISION: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", gasket_dev->hardware_revision); + ret = snprintf(buf, PAGE_SIZE, "%d\n", + gasket_dev->hardware_revision); break; case ATTR_PCI_ADDRESS: ret = snprintf(buf, PAGE_SIZE, "%s\n", gasket_dev->kobj_name); break; case ATTR_STATUS: - ret = snprintf( - buf, PAGE_SIZE, "%s\n", - gasket_num_name_lookup( - gasket_dev->status, gasket_status_name_table)); + ret = snprintf(buf, PAGE_SIZE, "%s\n", + gasket_num_name_lookup(gasket_dev->status, + gasket_status_name_table)); break; case ATTR_IS_DEVICE_OWNED: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", - gasket_dev->dev_info.ownership.is_owned); + ret = snprintf(buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.is_owned); break; case ATTR_DEVICE_OWNER: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", - gasket_dev->dev_info.ownership.owner); + ret = snprintf(buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.owner); break; case ATTR_WRITE_OPEN_COUNT: - ret = snprintf( - buf, PAGE_SIZE, "%d\n", - gasket_dev->dev_info.ownership.write_open_count); + ret = snprintf(buf, PAGE_SIZE, "%d\n", + gasket_dev->dev_info.ownership.write_open_count); break; case ATTR_RESET_COUNT: ret = snprintf(buf, PAGE_SIZE, "%d\n", gasket_dev->reset_count); break; case ATTR_USER_MEM_RANGES: for (i = 0; i < GASKET_NUM_BARS; ++i) { - current_written = gasket_write_mappable_regions( - buf, driver_desc, i); + current_written = + gasket_write_mappable_regions(buf, driver_desc, + i); buf += current_written; ret += current_written; } @@ -622,9 +620,9 @@ static const struct gasket_sysfs_attribute gasket_sysfs_generic_attrs[] = { }; /* Add a char device and related info. */ -static int gasket_add_cdev( - struct gasket_cdev_info *dev_info, - const struct file_operations *file_ops, struct module *owner) +static int gasket_add_cdev(struct gasket_cdev_info *dev_info, + const struct file_operations *file_ops, + struct module *owner) { int ret; @@ -672,8 +670,8 @@ static void gasket_disable_dev(struct gasket_dev *gasket_dev) * Precondition: Called with g_mutex held (to avoid a race on return). * Returns NULL if no matching device was found. */ -static struct gasket_internal_desc *lookup_internal_desc( - struct pci_dev *pci_dev) +static struct gasket_internal_desc * +lookup_internal_desc(struct pci_dev *pci_dev) { int i; @@ -693,9 +691,9 @@ static struct gasket_internal_desc *lookup_internal_desc( * that the provided descriptor/range is of adequate size to hold the range to * be mapped. */ -static bool gasket_mmap_has_permissions( - struct gasket_dev *gasket_dev, struct vm_area_struct *vma, - int bar_permissions) +static bool gasket_mmap_has_permissions(struct gasket_dev *gasket_dev, + struct vm_area_struct *vma, + int bar_permissions) { int requested_permissions; /* Always allow sysadmin to access. */ @@ -735,8 +733,9 @@ static bool gasket_mmap_has_permissions( * Verifies that the input address is within the region allocated to coherent * buffer. */ -static bool gasket_is_coherent_region( - const struct gasket_driver_desc *driver_desc, ulong address) +static bool +gasket_is_coherent_region(const struct gasket_driver_desc *driver_desc, + ulong address) { struct gasket_coherent_buffer_desc coh_buff_desc = driver_desc->coherent_buffer_description; @@ -750,8 +749,8 @@ static bool gasket_is_coherent_region( return false; } -static int gasket_get_bar_index( - const struct gasket_dev *gasket_dev, ulong phys_addr) +static int gasket_get_bar_index(const struct gasket_dev *gasket_dev, + ulong phys_addr) { int i; const struct gasket_driver_desc *driver_desc; @@ -786,10 +785,11 @@ static int gasket_get_bar_index( * * Returns true if there's anything to map, and false otherwise. */ -static bool gasket_mm_get_mapping_addrs( - const struct gasket_mappable_region *region, ulong bar_offset, - ulong requested_length, struct gasket_mappable_region *mappable_region, - ulong *virt_offset) +static bool +gasket_mm_get_mapping_addrs(const struct gasket_mappable_region *region, + ulong bar_offset, ulong requested_length, + struct gasket_mappable_region *mappable_region, + ulong *virt_offset) { ulong range_start = region->start; ulong range_length = region->length_bytes; @@ -835,8 +835,8 @@ static bool gasket_mm_get_mapping_addrs( */ mappable_region->start = bar_offset; *virt_offset = 0; - mappable_region->length_bytes = min( - requested_length, range_end - bar_offset); + mappable_region->length_bytes = + min(requested_length, range_end - bar_offset); return true; } @@ -852,9 +852,9 @@ static bool gasket_mm_get_mapping_addrs( * The offset is written into bar_offset on success. * Returns zero on success, anything else on error. */ -static int gasket_mm_vma_bar_offset( - const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma, - ulong *bar_offset) +static int gasket_mm_vma_bar_offset(const struct gasket_dev *gasket_dev, + const struct vm_area_struct *vma, + ulong *bar_offset) { ulong raw_offset; int bar_index; @@ -877,9 +877,9 @@ static int gasket_mm_vma_bar_offset( return 0; } -int gasket_mm_unmap_region( - const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, - const struct gasket_mappable_region *map_region) +int gasket_mm_unmap_region(const struct gasket_dev *gasket_dev, + struct vm_area_struct *vma, + const struct gasket_mappable_region *map_region) { ulong bar_offset; ulong virt_offset; @@ -893,9 +893,9 @@ int gasket_mm_unmap_region( if (ret) return ret; - if (!gasket_mm_get_mapping_addrs( - map_region, bar_offset, vma->vm_end - vma->vm_start, - &mappable_region, &virt_offset)) + if (!gasket_mm_get_mapping_addrs(map_region, bar_offset, + vma->vm_end - vma->vm_start, + &mappable_region, &virt_offset)) return 1; /* @@ -904,18 +904,17 @@ int gasket_mm_unmap_region( * * Next multiple of y: ceil_div(x, y) * y */ - zap_vma_ptes( - vma, vma->vm_start + virt_offset, - DIV_ROUND_UP(mappable_region.length_bytes, PAGE_SIZE) * - PAGE_SIZE); + zap_vma_ptes(vma, vma->vm_start + virt_offset, + DIV_ROUND_UP(mappable_region.length_bytes, PAGE_SIZE) * + PAGE_SIZE); return 0; } EXPORT_SYMBOL(gasket_mm_unmap_region); /* Maps a virtual address + range to a physical offset of a BAR. */ -static enum do_map_region_status do_map_region( - const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, - struct gasket_mappable_region *mappable_region) +static enum do_map_region_status +do_map_region(const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, + struct gasket_mappable_region *mappable_region) { /* Maximum size of a single call to io_remap_pfn_range. */ /* I pulled this number out of thin air. */ @@ -944,10 +943,9 @@ static enum do_map_region_status do_map_region( virt_base = vma->vm_start + virt_offset; bar_index = - gasket_get_bar_index( - gasket_dev, - (vma->vm_pgoff << PAGE_SHIFT) + - driver_desc->legacy_mmap_address_offset); + gasket_get_bar_index(gasket_dev, + (vma->vm_pgoff << PAGE_SHIFT) + + driver_desc->legacy_mmap_address_offset); phys_base = gasket_dev->bar_data[bar_index].phys_base + phys_offset; while (mapped_bytes < map_length) { /* @@ -957,10 +955,10 @@ static enum do_map_region_status do_map_region( chunk_size = min(max_chunk_size, map_length - mapped_bytes); cond_resched(); - ret = io_remap_pfn_range( - vma, virt_base + mapped_bytes, - (phys_base + mapped_bytes) >> PAGE_SHIFT, - chunk_size, vma->vm_page_prot); + ret = io_remap_pfn_range(vma, virt_base + mapped_bytes, + (phys_base + mapped_bytes) >> + PAGE_SHIFT, chunk_size, + vma->vm_page_prot); if (ret) { dev_err(gasket_dev->dev, "Error remapping PFN range.\n"); @@ -984,8 +982,8 @@ fail: } /* Map a region of coherent memory. */ -static int gasket_mmap_coherent( - struct gasket_dev *gasket_dev, struct vm_area_struct *vma) +static int gasket_mmap_coherent(struct gasket_dev *gasket_dev, + struct vm_area_struct *vma) { const struct gasket_driver_desc *driver_desc = gasket_dev->internal_desc->driver_desc; @@ -1008,10 +1006,9 @@ static int gasket_mmap_coherent( vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - ret = remap_pfn_range( - vma, vma->vm_start, - (gasket_dev->coherent_buffer.phys_base) >> PAGE_SHIFT, - requested_length, vma->vm_page_prot); + ret = remap_pfn_range(vma, vma->vm_start, + (gasket_dev->coherent_buffer.phys_base) >> + PAGE_SHIFT, requested_length, vma->vm_page_prot); if (ret) { dev_err(gasket_dev->dev, "Error remapping PFN range err=%d.\n", ret); @@ -1022,9 +1019,9 @@ static int gasket_mmap_coherent( /* Record the user virtual to dma_address mapping that was * created by the kernel. */ - gasket_set_user_virt( - gasket_dev, requested_length, - gasket_dev->coherent_buffer.phys_base, vma->vm_start); + gasket_set_user_virt(gasket_dev, requested_length, + gasket_dev->coherent_buffer.phys_base, + vma->vm_start); return 0; } @@ -1058,8 +1055,8 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) raw_offset = (vma->vm_pgoff << PAGE_SHIFT) + driver_desc->legacy_mmap_address_offset; vma_size = vma->vm_end - vma->vm_start; - trace_gasket_mmap_entry( - gasket_dev->dev_info.name, raw_offset, vma_size); + trace_gasket_mmap_entry(gasket_dev->dev_info.name, raw_offset, + vma_size); /* * Check if the raw offset is within a bar region. If not, check if it @@ -1103,8 +1100,10 @@ static int gasket_mmap(struct file *filp, struct vm_area_struct *vma) } if (driver_desc->get_mappable_regions_cb) { - ret = driver_desc->get_mappable_regions_cb( - gasket_dev, bar_index, &map_regions, &num_map_regions); + ret = driver_desc->get_mappable_regions_cb(gasket_dev, + bar_index, + &map_regions, + &num_map_regions); if (ret) return ret; } else { @@ -1231,8 +1230,8 @@ static int gasket_open(struct inode *inode, struct file *filp) /* If the node is not owned, assign it to the current TGID. */ if (!ownership->is_owned) { - ret = gasket_check_and_invoke_callback_nolock( - gasket_dev, driver_desc->device_open_cb); + ret = gasket_check_and_invoke_callback_nolock(gasket_dev, + driver_desc->device_open_cb); if (ret) { dev_err(gasket_dev->dev, "Error in device open cb: %d\n", ret); @@ -1298,16 +1297,14 @@ static int gasket_release(struct inode *inode, struct file *file) driver_desc->device_reset_cb(gasket_dev, 0); for (i = 0; i < driver_desc->num_page_tables; ++i) { - gasket_page_table_unmap_all( - gasket_dev->page_table[i]); - gasket_page_table_garbage_collect( - gasket_dev->page_table[i]); + gasket_page_table_unmap_all(gasket_dev->page_table[i]); + gasket_page_table_garbage_collect(gasket_dev->page_table[i]); gasket_free_coherent_memory_all(gasket_dev, i); } /* Closes device, enters power save. */ - gasket_check_and_invoke_callback_nolock( - gasket_dev, driver_desc->device_close_cb); + gasket_check_and_invoke_callback_nolock(gasket_dev, + driver_desc->device_close_cb); } } @@ -1367,21 +1364,21 @@ static const struct file_operations gasket_file_ops = { }; /* Perform final init and marks the device as active. */ -static int gasket_enable_dev( - struct gasket_internal_desc *internal_desc, - struct gasket_dev *gasket_dev) +static int gasket_enable_dev(struct gasket_internal_desc *internal_desc, + struct gasket_dev *gasket_dev) { int tbl_idx; int ret; const struct gasket_driver_desc *driver_desc = internal_desc->driver_desc; - ret = gasket_interrupt_init( - gasket_dev, driver_desc->name, - driver_desc->interrupt_type, driver_desc->interrupts, - driver_desc->num_interrupts, driver_desc->interrupt_pack_width, - driver_desc->interrupt_bar_index, - driver_desc->wire_interrupt_offsets); + ret = gasket_interrupt_init(gasket_dev, driver_desc->name, + driver_desc->interrupt_type, + driver_desc->interrupts, + driver_desc->num_interrupts, + driver_desc->interrupt_pack_width, + driver_desc->interrupt_bar_index, + driver_desc->wire_interrupt_offsets); if (ret) { dev_err(gasket_dev->dev, "Critical failure to allocate interrupts: %d\n", ret); @@ -1392,12 +1389,11 @@ static int gasket_enable_dev( for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) { dev_dbg(gasket_dev->dev, "Initializing page table %d.\n", tbl_idx); - ret = gasket_page_table_init( - &gasket_dev->page_table[tbl_idx], - &gasket_dev->bar_data[ - driver_desc->page_table_bar_index], - &driver_desc->page_table_configs[tbl_idx], - gasket_dev->dev, gasket_dev->pci_dev); + ret = gasket_page_table_init(&gasket_dev->page_table[tbl_idx], + &gasket_dev->bar_data[driver_desc->page_table_bar_index], + &driver_desc->page_table_configs[tbl_idx], + gasket_dev->dev, + gasket_dev->pci_dev); if (ret) { dev_err(gasket_dev->dev, "Couldn't init page table %d: %d\n", @@ -1415,8 +1411,8 @@ static int gasket_enable_dev( * hardware_revision_cb returns a positive integer (the rev) if * successful.) */ - ret = check_and_invoke_callback( - gasket_dev, driver_desc->hardware_revision_cb); + ret = check_and_invoke_callback(gasket_dev, + driver_desc->hardware_revision_cb); if (ret < 0) { dev_err(gasket_dev->dev, "Error getting hardware revision: %d\n", ret); @@ -1436,8 +1432,8 @@ static int gasket_enable_dev( if (gasket_dev->status == GASKET_STATUS_DEAD) dev_err(gasket_dev->dev, "Device reported as unhealthy.\n"); - ret = gasket_add_cdev( - &gasket_dev->dev_info, &gasket_file_ops, driver_desc->module); + ret = gasket_add_cdev(&gasket_dev->dev_info, &gasket_file_ops, + driver_desc->module); if (ret) return ret; @@ -1452,8 +1448,8 @@ static int gasket_enable_dev( * * Returns 0 if successful and a negative value otherwise. */ -static int gasket_pci_probe( - struct pci_dev *pci_dev, const struct pci_device_id *id) +static int gasket_pci_probe(struct pci_dev *pci_dev, + const struct pci_device_id *id) { int ret; const char *kobj_name = dev_name(&pci_dev->dev); @@ -1497,8 +1493,8 @@ static int gasket_pci_probe( goto fail2; } - ret = gasket_sysfs_create_mapping( - gasket_dev->dev_info.device, gasket_dev); + ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device, + gasket_dev); if (ret) goto fail3; @@ -1513,13 +1509,13 @@ static int gasket_pci_probe( "Cannot create sysfs pci link: %d\n", ret); goto fail3; } - ret = gasket_sysfs_create_entries( - gasket_dev->dev_info.device, gasket_sysfs_generic_attrs); + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, + gasket_sysfs_generic_attrs); if (ret) goto fail4; - ret = check_and_invoke_callback( - gasket_dev, driver_desc->sysfs_setup_cb); + ret = check_and_invoke_callback(gasket_dev, + driver_desc->sysfs_setup_cb); if (ret) { dev_err(gasket_dev->dev, "Error in sysfs setup cb: %d\n", ret); goto fail5; @@ -1611,8 +1607,8 @@ static void gasket_pci_remove(struct pci_dev *pci_dev) * * The table must have a NULL name pointer at the end. */ -const char *gasket_num_name_lookup( - uint num, const struct gasket_num_name *table) +const char *gasket_num_name_lookup(uint num, + const struct gasket_num_name *table) { uint i = 0; @@ -1677,8 +1673,8 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) } EXPORT_SYMBOL(gasket_reset_nolock); -gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( - struct gasket_dev *gasket_dev) +gasket_ioctl_permissions_cb_t +gasket_get_ioctl_permissions_cb(struct gasket_dev *gasket_dev) { return gasket_dev->internal_desc->driver_desc->ioctl_permissions_cb; } @@ -1713,9 +1709,9 @@ struct device *gasket_get_device(struct gasket_dev *dev) * Description: Busy waits for a specific combination of bits to be set on a * Gasket register. **/ -int gasket_wait_with_reschedule( - struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, - uint max_retries, u64 delay_ms) +int gasket_wait_with_reschedule(struct gasket_dev *gasket_dev, int bar, + u64 offset, u64 mask, u64 val, + uint max_retries, u64 delay_ms) { uint retries = 0; u64 tmp; @@ -1797,17 +1793,17 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) * depends on KBUILD_MODNAME, and this is a shared file. */ pr_debug("Registering PCI driver.\n"); - ret = __pci_register_driver( - &internal->pci, driver_desc->module, driver_desc->name); + ret = __pci_register_driver(&internal->pci, driver_desc->module, + driver_desc->name); if (ret) { pr_err("cannot register pci driver [ret=%d]\n", ret); goto fail1; } pr_debug("Registering char driver.\n"); - ret = register_chrdev_region( - MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, - driver_desc->name); + ret = register_chrdev_region(MKDEV(driver_desc->major, + driver_desc->minor), GASKET_DEV_MAX, + driver_desc->name); if (ret) { pr_err("cannot register char driver [ret=%d]\n", ret); goto fail2; @@ -1853,8 +1849,8 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) return; } - unregister_chrdev_region( - MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX); + unregister_chrdev_region(MKDEV(driver_desc->major, driver_desc->minor), + GASKET_DEV_MAX); pci_unregister_driver(&internal_desc->pci); diff --git a/gasket_core.h b/gasket_core.h index 8bd431a..713bf42 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -318,11 +318,11 @@ struct gasket_dev { }; /* Type of the ioctl handler callback. */ -typedef long (*gasket_ioctl_handler_cb_t) - (struct file *file, uint cmd, void __user *argp); +typedef long (*gasket_ioctl_handler_cb_t)(struct file *file, uint cmd, + void __user *argp); /* Type of the ioctl permissions check callback. See below. */ -typedef int (*gasket_ioctl_permissions_cb_t)( - struct file *filp, uint cmd, void __user *argp); +typedef int (*gasket_ioctl_permissions_cb_t)(struct file *filp, uint cmd, + void __user *argp); /* * Device type descriptor. @@ -457,8 +457,8 @@ struct gasket_driver_desc { * descriptor for an open file is closed. This call is intended to * handle any per-user or per-fd cleanup. */ - int (*device_release_cb)( - struct gasket_dev *gasket_dev, struct file *file); + int (*device_release_cb)(struct gasket_dev *gasket_dev, + struct file *file); /* * device_close_cb: Callback for when a device node is closed for the @@ -527,10 +527,10 @@ struct gasket_driver_desc { * information is then compared to mmap request to determine which * regions to actually map. */ - int (*get_mappable_regions_cb)( - struct gasket_dev *gasket_dev, int bar_index, - struct gasket_mappable_region **mappable_regions, - int *num_mappable_regions); + int (*get_mappable_regions_cb)(struct gasket_dev *gasket_dev, + int bar_index, + struct gasket_mappable_region **mappable_regions, + int *num_mappable_regions); /* * ioctl_permissions_cb: Check permissions for generic ioctls. @@ -631,16 +631,16 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type); */ /* Unmaps the specified mappable region from a VMA. */ -int gasket_mm_unmap_region( - const struct gasket_dev *gasket_dev, struct vm_area_struct *vma, - const struct gasket_mappable_region *map_region); +int gasket_mm_unmap_region(const struct gasket_dev *gasket_dev, + struct vm_area_struct *vma, + const struct gasket_mappable_region *map_region); /* * Get the ioctl permissions callback. * @gasket_dev: Gasket device structure. */ -gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( - struct gasket_dev *gasket_dev); +gasket_ioctl_permissions_cb_t +gasket_get_ioctl_permissions_cb(struct gasket_dev *gasket_dev); /** * Lookup a name by number in a num_name table. @@ -648,37 +648,37 @@ gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb( * @table: Array of num_name structures, the table for the lookup. * */ -const char *gasket_num_name_lookup( - uint num, const struct gasket_num_name *table); +const char *gasket_num_name_lookup(uint num, + const struct gasket_num_name *table); /* Handy inlines */ -static inline ulong gasket_dev_read_64( - struct gasket_dev *gasket_dev, int bar, ulong location) +static inline ulong gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar, + ulong location) { return readq(&gasket_dev->bar_data[bar].virt_base[location]); } -static inline void gasket_dev_write_64( - struct gasket_dev *dev, u64 value, int bar, ulong location) +static inline void gasket_dev_write_64(struct gasket_dev *dev, u64 value, + int bar, ulong location) { writeq(value, &dev->bar_data[bar].virt_base[location]); } -static inline void gasket_dev_write_32( - struct gasket_dev *dev, u32 value, int bar, ulong location) +static inline void gasket_dev_write_32(struct gasket_dev *dev, u32 value, + int bar, ulong location) { writel(value, &dev->bar_data[bar].virt_base[location]); } -static inline u32 gasket_dev_read_32( - struct gasket_dev *dev, int bar, ulong location) +static inline u32 gasket_dev_read_32(struct gasket_dev *dev, int bar, + ulong location) { return readl(&dev->bar_data[bar].virt_base[location]); } -static inline void gasket_read_modify_write_64( - struct gasket_dev *dev, int bar, ulong location, u64 value, - u64 mask_width, u64 mask_shift) +static inline void gasket_read_modify_write_64(struct gasket_dev *dev, int bar, + ulong location, u64 value, + u64 mask_width, u64 mask_shift) { u64 mask, tmp; @@ -688,9 +688,9 @@ static inline void gasket_read_modify_write_64( gasket_dev_write_64(dev, tmp, bar, location); } -static inline void gasket_read_modify_write_32( - struct gasket_dev *dev, int bar, ulong location, u32 value, - u32 mask_width, u32 mask_shift) +static inline void gasket_read_modify_write_32(struct gasket_dev *dev, int bar, + ulong location, u32 value, + u32 mask_width, u32 mask_shift) { u32 mask, tmp; @@ -707,8 +707,8 @@ const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev); struct device *gasket_get_device(struct gasket_dev *dev); /* Helper function, Asynchronous waits on a given set of bits. */ -int gasket_wait_with_reschedule( - struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val, - uint max_retries, u64 delay_ms); +int gasket_wait_with_reschedule(struct gasket_dev *gasket_dev, int bar, + u64 offset, u64 mask, u64 val, + uint max_retries, u64 delay_ms); #endif /* __GASKET_CORE_H__ */ From 18e8ceb69bf6f38dc5f931dfbf7e93ae96824ac7 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:40 -0700 Subject: [PATCH 111/209] staging: gasket: ioctl: fix function param line continuation style Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); Many of these TODO items were previously cleaned up during the conversion to standard logging functions. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_ioctl.c | 60 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 134d45a..d3397cc 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -79,12 +79,12 @@ static int gasket_read_simple_page_table_size( if (ibuf.page_table_index >= gasket_dev->num_page_tables) return -EFAULT; - ibuf.size = gasket_page_table_num_simple_entries( - gasket_dev->page_table[ibuf.page_table_index]); + ibuf.size = + gasket_page_table_num_simple_entries(gasket_dev->page_table[ibuf.page_table_index]); - trace_gasket_ioctl_page_table_data( - ibuf.page_table_index, ibuf.size, ibuf.host_address, - ibuf.device_address); + trace_gasket_ioctl_page_table_data(ibuf.page_table_index, ibuf.size, + ibuf.host_address, + ibuf.device_address); if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; @@ -138,21 +138,21 @@ static int gasket_map_buffers(struct gasket_dev *gasket_dev, if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; - trace_gasket_ioctl_page_table_data( - ibuf.page_table_index, ibuf.size, ibuf.host_address, - ibuf.device_address); + trace_gasket_ioctl_page_table_data(ibuf.page_table_index, ibuf.size, + ibuf.host_address, + ibuf.device_address); if (ibuf.page_table_index >= gasket_dev->num_page_tables) return -EFAULT; - if (gasket_page_table_are_addrs_bad( - gasket_dev->page_table[ibuf.page_table_index], - ibuf.host_address, ibuf.device_address, ibuf.size)) + if (gasket_page_table_are_addrs_bad(gasket_dev->page_table[ibuf.page_table_index], + ibuf.host_address, + ibuf.device_address, ibuf.size)) return -EINVAL; - return gasket_page_table_map( - gasket_dev->page_table[ibuf.page_table_index], - ibuf.host_address, ibuf.device_address, ibuf.size / PAGE_SIZE); + return gasket_page_table_map(gasket_dev->page_table[ibuf.page_table_index], + ibuf.host_address, ibuf.device_address, + ibuf.size / PAGE_SIZE); } /* Unmap a userspace buffer from a device virtual address. */ @@ -164,16 +164,15 @@ static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; - trace_gasket_ioctl_page_table_data( - ibuf.page_table_index, ibuf.size, ibuf.host_address, - ibuf.device_address); + trace_gasket_ioctl_page_table_data(ibuf.page_table_index, ibuf.size, + ibuf.host_address, + ibuf.device_address); if (ibuf.page_table_index >= gasket_dev->num_page_tables) return -EFAULT; - if (gasket_page_table_is_dev_addr_bad( - gasket_dev->page_table[ibuf.page_table_index], - ibuf.device_address, ibuf.size)) + if (gasket_page_table_is_dev_addr_bad(gasket_dev->page_table[ibuf.page_table_index], + ibuf.device_address, ibuf.size)) return -EINVAL; gasket_page_table_unmap(gasket_dev->page_table[ibuf.page_table_index], @@ -197,8 +196,8 @@ static int gasket_config_coherent_allocator( sizeof(struct gasket_coherent_alloc_config_ioctl))) return -EFAULT; - trace_gasket_ioctl_config_coherent_allocator( - ibuf.enable, ibuf.size, ibuf.dma_address); + trace_gasket_ioctl_config_coherent_allocator(ibuf.enable, ibuf.size, + ibuf.dma_address); if (ibuf.page_table_index >= gasket_dev->num_page_tables) return -EFAULT; @@ -207,13 +206,13 @@ static int gasket_config_coherent_allocator( return -ENOMEM; if (ibuf.enable == 0) { - ret = gasket_free_coherent_memory( - gasket_dev, ibuf.size, ibuf.dma_address, - ibuf.page_table_index); + ret = gasket_free_coherent_memory(gasket_dev, ibuf.size, + ibuf.dma_address, + ibuf.page_table_index); } else { - ret = gasket_alloc_coherent_memory( - gasket_dev, ibuf.size, &ibuf.dma_address, - ibuf.page_table_index); + ret = gasket_alloc_coherent_memory(gasket_dev, ibuf.size, + &ibuf.dma_address, + ibuf.page_table_index); } if (ret) return ret; @@ -313,8 +312,9 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) break; case GASKET_IOCTL_CLEAR_EVENTFD: trace_gasket_ioctl_integer_data(arg); - retval = gasket_interrupt_clear_eventfd( - gasket_dev->interrupt_data, (int)arg); + retval = + gasket_interrupt_clear_eventfd(gasket_dev->interrupt_data, + (int)arg); break; case GASKET_IOCTL_PARTITION_PAGE_TABLE: trace_gasket_ioctl_integer_data(arg); From 1babb38387c5fc971642575c931d82d71c7c2926 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:41 -0700 Subject: [PATCH 112/209] staging: gasket: page table: fix function param line continuation style Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); Many of these TODO items were previously cleaned up during the conversion to standard logging functions. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 206 ++++++++++++++++++++++---------------------- gasket_page_table.h | 33 ++++--- 2 files changed, 120 insertions(+), 119 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index aa036b2..13e1d09 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -215,11 +215,10 @@ struct gasket_page_table { }; /* See gasket_page_table.h for description. */ -int gasket_page_table_init( - struct gasket_page_table **ppg_tbl, - const struct gasket_bar_data *bar_data, - const struct gasket_page_table_config *page_table_config, - struct device *device, struct pci_dev *pci_dev) +int gasket_page_table_init(struct gasket_page_table **ppg_tbl, + const struct gasket_bar_data *bar_data, + const struct gasket_page_table_config *page_table_config, + struct device *device, struct pci_dev *pci_dev) { ulong bytes; struct gasket_page_table *pg_tbl; @@ -276,10 +275,10 @@ int gasket_page_table_init( pg_tbl->extended_flag = 0; } pg_tbl->num_active_pages = 0; - pg_tbl->base_slot = (u64 __iomem *)&( - bar_data->virt_base[page_table_config->base_reg]); - pg_tbl->extended_offset_reg = (u64 __iomem *)&( - bar_data->virt_base[page_table_config->extended_reg]); + pg_tbl->base_slot = + (u64 __iomem *)&bar_data->virt_base[page_table_config->base_reg]; + pg_tbl->extended_offset_reg = + (u64 __iomem *)&bar_data->virt_base[page_table_config->extended_reg]; pg_tbl->device = get_device(device); pg_tbl->pci_dev = pci_dev_get(pci_dev); @@ -292,8 +291,8 @@ int gasket_page_table_init( * Check if a range of PTEs is free. * The page table mutex must be held by the caller. */ -static bool gasket_is_pte_range_free( - struct gasket_page_table_entry *ptes, uint num_entries) +static bool gasket_is_pte_range_free(struct gasket_page_table_entry *ptes, + uint num_entries) { int i; @@ -309,9 +308,9 @@ static bool gasket_is_pte_range_free( * Free a second level page [sub]table. * The page table mutex must be held before this call. */ -static void gasket_free_extended_subtable( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, - u64 __iomem *slot) +static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl, + struct gasket_page_table_entry *pte, + u64 __iomem *slot) { /* Release the page table from the driver */ pte->status = PTE_FREE; @@ -337,8 +336,8 @@ static void gasket_free_extended_subtable( * Actually perform collection. * The page table mutex must be held by the caller. */ -static void gasket_page_table_garbage_collect_nolock( - struct gasket_page_table *pg_tbl) +static void +gasket_page_table_garbage_collect_nolock(struct gasket_page_table *pg_tbl) { struct gasket_page_table_entry *pte; u64 __iomem *slot; @@ -351,10 +350,10 @@ static void gasket_page_table_garbage_collect_nolock( pte < pg_tbl->entries + pg_tbl->config.total_entries; pte++, slot++) { if (pte->status == PTE_INUSE) { - if (gasket_is_pte_range_free( - pte->sublevel, GASKET_PAGES_PER_SUBTABLE)) - gasket_free_extended_subtable( - pg_tbl, pte, slot); + if (gasket_is_pte_range_free(pte->sublevel, + GASKET_PAGES_PER_SUBTABLE)) + gasket_free_extended_subtable(pg_tbl, pte, + slot); } } } @@ -384,8 +383,8 @@ void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl) } /* See gasket_page_table.h for description. */ -int gasket_page_table_partition( - struct gasket_page_table *pg_tbl, uint num_simple_entries) +int gasket_page_table_partition(struct gasket_page_table *pg_tbl, + uint num_simple_entries) { int i, start; @@ -445,10 +444,10 @@ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) * an extended mapping, these will be within a second-level page table * allocated by the host and so must have their __iomem attribute casted away. */ -static int gasket_perform_mapping( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, - u64 __iomem *slots, ulong host_addr, uint num_pages, - int is_simple_mapping) +static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, + struct gasket_page_table_entry *ptes, + u64 __iomem *slots, ulong host_addr, + uint num_pages, int is_simple_mapping) { int ret; ulong offset; @@ -470,8 +469,8 @@ static int gasket_perform_mapping( ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + off + i * PAGE_SIZE; } else { - ret = get_user_pages_fast( - page_addr - offset, 1, 1, &page); + ret = get_user_pages_fast(page_addr - offset, 1, 1, + &page); if (ret <= 0) { dev_err(pg_tbl->device, @@ -532,8 +531,8 @@ static int gasket_perform_mapping( * Return the index of the page for the address in the simple table. * Does not perform validity checking. */ -static int gasket_simple_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr) +static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl, + ulong dev_addr) { return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & (pg_tbl->config.total_entries - 1); @@ -543,8 +542,8 @@ static int gasket_simple_page_idx( * Return the level 0 page index for the given address. * Does not perform validity checking. */ -static ulong gasket_extended_lvl0_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr) +static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, + ulong dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); @@ -554,8 +553,8 @@ static ulong gasket_extended_lvl0_page_idx( * Return the level 1 page index for the given address. * Does not perform validity checking. */ -static ulong gasket_extended_lvl1_page_idx( - struct gasket_page_table *pg_tbl, ulong dev_addr) +static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl, + ulong dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & (GASKET_PAGES_PER_SUBTABLE - 1); @@ -565,12 +564,12 @@ static ulong gasket_extended_lvl1_page_idx( * Allocate page table entries in a simple table. * The page table mutex must be held by the caller. */ -static int gasket_alloc_simple_entries( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_pages) { - if (!gasket_is_pte_range_free( - pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr), - num_pages)) + if (!gasket_is_pte_range_free(pg_tbl->entries + + gasket_simple_page_idx(pg_tbl, dev_addr), + num_pages)) return -EBUSY; return 0; @@ -593,9 +592,10 @@ static bool gasket_release_page(struct page *page) * Unmap and release mapped pages. * The page table mutex must be held by the caller. */ -static void gasket_perform_unmapping( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, - u64 __iomem *slots, uint num_pages, int is_simple_mapping) +static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, + struct gasket_page_table_entry *ptes, + u64 __iomem *slots, uint num_pages, + int is_simple_mapping) { int i; /* @@ -631,8 +631,8 @@ static void gasket_perform_unmapping( * Unmap and release pages mapped to simple addresses. * The page table mutex must be held by the caller. */ -static void gasket_unmap_simple_pages( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_pages) { uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); @@ -644,8 +644,8 @@ static void gasket_unmap_simple_pages( * Unmap and release buffers to extended addresses. * The page table mutex must be held by the caller. */ -static void gasket_unmap_extended_pages( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_pages) { uint slot_idx, remain, len; struct gasket_page_table_entry *pte; @@ -663,9 +663,9 @@ static void gasket_unmap_extended_pages( if (pte->status == PTE_INUSE) { slot_base = (u64 __iomem *)(page_address(pte->page) + pte->offset); - gasket_perform_unmapping( - pg_tbl, pte->sublevel + slot_idx, - slot_base + slot_idx, len, 0); + gasket_perform_unmapping(pg_tbl, + pte->sublevel + slot_idx, + slot_base + slot_idx, len, 0); } remain -= len; @@ -675,8 +675,8 @@ static void gasket_unmap_extended_pages( } /* Evaluates to nonzero if the specified virtual address is simple. */ -static inline bool gasket_addr_is_simple( - struct gasket_page_table *pg_tbl, ulong addr) +static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, + ulong addr) { return !((addr) & (pg_tbl)->extended_flag); } @@ -693,9 +693,9 @@ static inline bool gasket_addr_is_simple( * Extended page 1000, offset 511: * Input (1, 1000, 512), Output 0x8003E81FF */ -static ulong gasket_components_to_dev_address( - struct gasket_page_table *pg_tbl, int is_simple, uint page_index, - uint offset) +static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, + int is_simple, uint page_index, + uint offset) { ulong lvl0_index, lvl1_index; @@ -726,15 +726,15 @@ static ulong gasket_components_to_dev_address( * and that the requested page range starts and ends within the set of * currently-partitioned simple pages. */ -static bool gasket_is_simple_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_pages) { ulong page_offset = dev_addr & (PAGE_SIZE - 1); ulong page_index = (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1); - if (gasket_components_to_dev_address( - pg_tbl, 1, page_index, page_offset) != dev_addr) { + if (gasket_components_to_dev_address(pg_tbl, 1, page_index, + page_offset) != dev_addr) { dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", dev_addr); return true; @@ -764,8 +764,8 @@ static bool gasket_is_simple_dev_addr_bad( * offset) and that the requested page range starts and ends within the set of * currently-partitioned extended pages. */ -static bool gasket_is_extended_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_pages) { /* Starting byte index of dev_addr into the first mapped page */ ulong page_offset = dev_addr & (PAGE_SIZE - 1); @@ -792,8 +792,8 @@ static bool gasket_is_extended_dev_addr_bad( num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) / GASKET_PAGES_PER_SUBTABLE; - if (gasket_components_to_dev_address( - pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { + if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx, + page_offset) != dev_addr) { dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", dev_addr); return true; @@ -821,8 +821,8 @@ static bool gasket_is_extended_dev_addr_bad( * Non-locking entry to unmapping routines. * The page table mutex must be held by the caller. */ -static void gasket_page_table_unmap_nolock( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_pages) { if (!num_pages) return; @@ -837,9 +837,9 @@ static void gasket_page_table_unmap_nolock( * Allocate and map pages to simple addresses. * If there is an error, no pages are mapped. */ -static int gasket_map_simple_pages( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) +static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, + ulong host_addr, ulong dev_addr, + uint num_pages) { int ret; uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); @@ -852,9 +852,9 @@ static int gasket_map_simple_pages( return ret; } - ret = gasket_perform_mapping( - pg_tbl, pg_tbl->entries + slot_idx, - pg_tbl->base_slot + slot_idx, host_addr, num_pages, 1); + ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx, + pg_tbl->base_slot + slot_idx, host_addr, + num_pages, 1); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); @@ -867,9 +867,9 @@ static int gasket_map_simple_pages( * Allocate a second level page table. * The page table mutex must be held by the caller. */ -static int gasket_alloc_extended_subtable( - struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte, - u64 __iomem *slot) +static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, + struct gasket_page_table_entry *pte, + u64 __iomem *slot) { ulong page_addr, subtable_bytes; dma_addr_t dma_addr; @@ -924,8 +924,8 @@ static int gasket_alloc_extended_subtable( * * The page table mutex must be held by the caller. */ -static int gasket_alloc_extended_entries( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries) +static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, + ulong dev_addr, uint num_entries) { int ret = 0; uint remain, subtable_slot_idx, len; @@ -951,8 +951,8 @@ static int gasket_alloc_extended_entries( return ret; } } else { - if (!gasket_is_pte_range_free( - pte->sublevel + subtable_slot_idx, len)) + if (!gasket_is_pte_range_free(pte->sublevel + + subtable_slot_idx, len)) return -EBUSY; } @@ -969,9 +969,9 @@ static int gasket_alloc_extended_entries( * gasket_map_extended_pages - Get and map buffers to extended addresses. * If there is an error, no pages are mapped. */ -static int gasket_map_extended_pages( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) +static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, + ulong host_addr, ulong dev_addr, + uint num_pages) { int ret; ulong dev_addr_end; @@ -1003,12 +1003,12 @@ static int gasket_map_extended_pages( slot_base = (u64 __iomem *)(page_address(pte->page) + pte->offset); - ret = gasket_perform_mapping( - pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, - host_addr, len, 0); + ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx, + slot_base + slot_idx, host_addr, + len, 0); if (ret) { - gasket_page_table_unmap_nolock( - pg_tbl, dev_addr, num_pages); + gasket_page_table_unmap_nolock(pg_tbl, dev_addr, + num_pages); return ret; } @@ -1029,9 +1029,8 @@ static int gasket_map_extended_pages( * * The page table mutex is held for the entire operation. */ -int gasket_page_table_map( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) +int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, + ulong dev_addr, uint num_pages) { int ret; @@ -1041,11 +1040,11 @@ int gasket_page_table_map( mutex_lock(&pg_tbl->mutex); if (gasket_addr_is_simple(pg_tbl, dev_addr)) { - ret = gasket_map_simple_pages( - pg_tbl, host_addr, dev_addr, num_pages); + ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr, + num_pages); } else { - ret = gasket_map_extended_pages( - pg_tbl, host_addr, dev_addr, num_pages); + ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr, + num_pages); } mutex_unlock(&pg_tbl->mutex); @@ -1067,8 +1066,8 @@ EXPORT_SYMBOL(gasket_page_table_map); * * The page table mutex is held for the entire operation. */ -void gasket_page_table_unmap( - struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages) +void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr, + uint num_pages) { if (!num_pages) return; @@ -1081,12 +1080,15 @@ EXPORT_SYMBOL(gasket_page_table_unmap); static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl) { - gasket_unmap_simple_pages( - pg_tbl, gasket_components_to_dev_address(pg_tbl, 1, 0, 0), - pg_tbl->num_simple_entries); - gasket_unmap_extended_pages( - pg_tbl, gasket_components_to_dev_address(pg_tbl, 0, 0, 0), - pg_tbl->num_extended_entries * GASKET_PAGES_PER_SUBTABLE); + gasket_unmap_simple_pages(pg_tbl, + gasket_components_to_dev_address(pg_tbl, 1, 0, + 0), + pg_tbl->num_simple_entries); + gasket_unmap_extended_pages(pg_tbl, + gasket_components_to_dev_address(pg_tbl, 0, + 0, 0), + pg_tbl->num_extended_entries * + GASKET_PAGES_PER_SUBTABLE); } /* See gasket_page_table.h for description. */ @@ -1189,8 +1191,8 @@ bool gasket_page_table_is_dev_addr_bad( } if (gasket_addr_is_simple(pg_tbl, dev_addr)) - return gasket_is_simple_dev_addr_bad( - pg_tbl, dev_addr, num_pages); + return gasket_is_simple_dev_addr_bad(pg_tbl, dev_addr, + num_pages); return gasket_is_extended_dev_addr_bad(pg_tbl, dev_addr, num_pages); } EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad); diff --git a/gasket_page_table.h b/gasket_page_table.h index 7655886..00c06f0 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -46,11 +46,10 @@ struct gasket_page_table; * * Returns 0 on success, a negative error code otherwise. */ -int gasket_page_table_init( - struct gasket_page_table **ppg_tbl, - const struct gasket_bar_data *bar_data, - const struct gasket_page_table_config *page_table_config, - struct device *device, struct pci_dev *pci_dev); +int gasket_page_table_init(struct gasket_page_table **ppg_tbl, + const struct gasket_bar_data *bar_data, + const struct gasket_page_table_config *page_table_config, + struct device *device, struct pci_dev *pci_dev); /* * Deallocate and cleanup page table data. @@ -77,8 +76,8 @@ void gasket_page_table_cleanup(struct gasket_page_table *page_table); * Returns 0 if successful, or non-zero if the page table entries * are not free. */ -int gasket_page_table_partition( - struct gasket_page_table *page_table, uint num_simple_entries); +int gasket_page_table_partition(struct gasket_page_table *page_table, + uint num_simple_entries); /* * Get and map [host] user space pages into device memory. @@ -106,8 +105,8 @@ int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, * * Description: The inverse of gasket_map_pages. Unmaps pages from the device. */ -void gasket_page_table_unmap( - struct gasket_page_table *page_table, ulong dev_addr, uint num_pages); +void gasket_page_table_unmap(struct gasket_page_table *page_table, + ulong dev_addr, uint num_pages); /* * Unmap ALL host pages from device memory. @@ -146,9 +145,9 @@ void gasket_page_table_garbage_collect(struct gasket_page_table *page_table); * Returns 0 if successful, -1 for an error. The page pointer * and offset are returned through the pointers, if successful. */ -int gasket_page_table_lookup_page( - struct gasket_page_table *page_table, ulong dev_addr, - struct page **page, ulong *poffset); +int gasket_page_table_lookup_page(struct gasket_page_table *page_table, + ulong dev_addr, struct page **page, + ulong *poffset); /* * Checks validity for input addrs and size. @@ -163,9 +162,9 @@ int gasket_page_table_lookup_page( * * Returns true if the mapping is bad, false otherwise. */ -bool gasket_page_table_are_addrs_bad( - struct gasket_page_table *page_table, ulong host_addr, ulong dev_addr, - ulong bytes); +bool gasket_page_table_are_addrs_bad(struct gasket_page_table *page_table, + ulong host_addr, ulong dev_addr, + ulong bytes); /* * Checks validity for input dev addr and size. @@ -179,8 +178,8 @@ bool gasket_page_table_are_addrs_bad( * * Returns true if the address is bad, false otherwise. */ -bool gasket_page_table_is_dev_addr_bad( - struct gasket_page_table *page_table, ulong dev_addr, ulong bytes); +bool gasket_page_table_is_dev_addr_bad(struct gasket_page_table *page_table, + ulong dev_addr, ulong bytes); /* * Gets maximum size for the given page table. From bfec1de4feee4acb0ab30f878f2d0d13518dbaec Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:42 -0700 Subject: [PATCH 113/209] staging: gasket: sysfs: fix function param line continuation style Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 26 +++++++++++++------------- gasket_sysfs.h | 32 ++++++++++++++++---------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index ef4eca0..a4bfca4 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -145,8 +145,8 @@ void gasket_sysfs_init(void) } } -int gasket_sysfs_create_mapping( - struct device *device, struct gasket_dev *gasket_dev) +int gasket_sysfs_create_mapping(struct device *device, + struct gasket_dev *gasket_dev) { struct gasket_sysfs_mapping *mapping; int map_idx = -1; @@ -210,8 +210,8 @@ int gasket_sysfs_create_mapping( return 0; } -int gasket_sysfs_create_entries( - struct device *device, const struct gasket_sysfs_attribute *attrs) +int gasket_sysfs_create_entries(struct device *device, + const struct gasket_sysfs_attribute *attrs) { int i; int ret; @@ -293,8 +293,8 @@ void gasket_sysfs_put_device_data(struct device *device, struct gasket_dev *dev) } EXPORT_SYMBOL(gasket_sysfs_put_device_data); -struct gasket_sysfs_attribute *gasket_sysfs_get_attr( - struct device *device, struct device_attribute *attr) +struct gasket_sysfs_attribute * +gasket_sysfs_get_attr(struct device *device, struct device_attribute *attr) { int i; int num_attrs; @@ -317,8 +317,8 @@ struct gasket_sysfs_attribute *gasket_sysfs_get_attr( } EXPORT_SYMBOL(gasket_sysfs_get_attr); -void gasket_sysfs_put_attr( - struct device *device, struct gasket_sysfs_attribute *attr) +void gasket_sysfs_put_attr(struct device *device, + struct gasket_sysfs_attribute *attr) { int i; int num_attrs; @@ -342,9 +342,9 @@ void gasket_sysfs_put_attr( } EXPORT_SYMBOL(gasket_sysfs_put_attr); -ssize_t gasket_sysfs_register_store( - struct device *device, struct device_attribute *attr, const char *buf, - size_t count) +ssize_t gasket_sysfs_register_store(struct device *device, + struct device_attribute *attr, + const char *buf, size_t count) { ulong parsed_value = 0; struct gasket_sysfs_mapping *mapping; @@ -386,8 +386,8 @@ ssize_t gasket_sysfs_register_store( gasket_attr->data.bar_address.offset); if (gasket_attr->write_callback) - gasket_attr->write_callback( - gasket_dev, gasket_attr, parsed_value); + gasket_attr->write_callback(gasket_dev, gasket_attr, + parsed_value); gasket_sysfs_put_attr(device, gasket_attr); put_mapping(mapping); diff --git a/gasket_sysfs.h b/gasket_sysfs.h index e9f4fad..f32eaf8 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -68,9 +68,9 @@ struct gasket_sysfs_attribute { * The callback should perform any logging necessary, as errors cannot * be returned from the callback. */ - void (*write_callback)( - struct gasket_dev *dev, struct gasket_sysfs_attribute *attr, - ulong value); + void (*write_callback)(struct gasket_dev *dev, + struct gasket_sysfs_attribute *attr, + ulong value); }; #define GASKET_SYSFS_RO(_name, _show_function, _attr_type) \ @@ -98,8 +98,8 @@ void gasket_sysfs_init(void); * If this function is not called before gasket_sysfs_create_entries, a warning * will be logged. */ -int gasket_sysfs_create_mapping( - struct device *device, struct gasket_dev *gasket_dev); +int gasket_sysfs_create_mapping(struct device *device, + struct gasket_dev *gasket_dev); /* * Creates bulk entries in sysfs. @@ -111,8 +111,8 @@ int gasket_sysfs_create_mapping( * gasket_sysfs_create_mapping had a legacy device, the entries will be created * for it, as well. */ -int gasket_sysfs_create_entries( - struct device *device, const struct gasket_sysfs_attribute *attrs); +int gasket_sysfs_create_entries(struct device *device, + const struct gasket_sysfs_attribute *attrs); /* * Removes a device mapping from the global table. @@ -141,8 +141,8 @@ struct gasket_dev *gasket_sysfs_get_device_data(struct device *device); * @device: Kernel device structure. * @dev: Gasket device descriptor (returned by gasket_sysfs_get_device_data). */ -void gasket_sysfs_put_device_data( - struct device *device, struct gasket_dev *gasket_dev); +void gasket_sysfs_put_device_data(struct device *device, + struct gasket_dev *gasket_dev); /* * Gasket-specific attribute lookup. @@ -155,8 +155,8 @@ void gasket_sysfs_put_device_data( * gasket_sysfs_get_device_data. While this reference is held, the underlying * device sysfs information/structure will remain valid/will not be deleted. */ -struct gasket_sysfs_attribute *gasket_sysfs_get_attr( - struct device *device, struct device_attribute *attr); +struct gasket_sysfs_attribute * +gasket_sysfs_get_attr(struct device *device, struct device_attribute *attr); /* * Releases a references to internal data. @@ -164,16 +164,16 @@ struct gasket_sysfs_attribute *gasket_sysfs_get_attr( * @attr: Gasket sysfs attribute descriptor (returned by * gasket_sysfs_get_attr). */ -void gasket_sysfs_put_attr( - struct device *device, struct gasket_sysfs_attribute *attr); +void gasket_sysfs_put_attr(struct device *device, + struct gasket_sysfs_attribute *attr); /* * Write to a register sysfs node. * @buf: NULL-terminated data being written. * @count: number of bytes in the "buf" argument. */ -ssize_t gasket_sysfs_register_store( - struct device *device, struct device_attribute *attr, const char *buf, - size_t count); +ssize_t gasket_sysfs_register_store(struct device *device, + struct device_attribute *attr, + const char *buf, size_t count); #endif /* __GASKET_SYSFS_H__ */ From 750452c7e28db63ec7add4c474027800245cbefc Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:43 -0700 Subject: [PATCH 114/209] staging: gasket: interrupt: fix function param line continuation style Fix multi-line alignment formatting to look like: int ret = long_function_name(device, VARIABLE1, VARIABLE2, VARIABLE3, VARIABLE4); Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 73 +++++++++++++++++++++------------------------- gasket_interrupt.h | 19 ++++++------ 2 files changed, 43 insertions(+), 49 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 3079b59..09c3d07 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -144,11 +144,10 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } mask = ~(0xFFFF << pack_shift); - value = gasket_dev_read_64( - gasket_dev, - interrupt_data->interrupt_bar_index, - interrupt_data->interrupts[i].reg) & - mask; + value = gasket_dev_read_64(gasket_dev, + interrupt_data->interrupt_bar_index, + interrupt_data->interrupts[i].reg); + value &= mask; value |= interrupt_data->interrupts[i].index << pack_shift; } @@ -187,8 +186,8 @@ static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) return IRQ_HANDLED; } -static int gasket_interrupt_msix_init( - struct gasket_interrupt_data *interrupt_data) +static int +gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data) { int ret = 1; int i; @@ -210,10 +209,9 @@ static int gasket_interrupt_msix_init( interrupt_data->msix_configured = 1; for (i = 0; i < interrupt_data->num_interrupts; i++) { - ret = request_irq( - interrupt_data->msix_entries[i].vector, - gasket_msix_interrupt_handler, 0, interrupt_data->name, - interrupt_data); + ret = request_irq(interrupt_data->msix_entries[i].vector, + gasket_msix_interrupt_handler, 0, + interrupt_data->name, interrupt_data); if (ret) { dev_err(&interrupt_data->pci_dev->dev, @@ -250,25 +248,23 @@ static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev) ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE + MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE; u32 mask = - gasket_dev_read_32( - gasket_dev, - gasket_dev->interrupt_data->interrupt_bar_index, - location); + gasket_dev_read_32(gasket_dev, + gasket_dev->interrupt_data->interrupt_bar_index, + location); if (!(mask & 1)) continue; /* Unmask the msix vector (clear 32 bits) */ - gasket_dev_write_32( - gasket_dev, 0, - gasket_dev->interrupt_data->interrupt_bar_index, - location); + gasket_dev_write_32(gasket_dev, 0, + gasket_dev->interrupt_data->interrupt_bar_index, + location); } #undef MSIX_VECTOR_SIZE #undef MSIX_MASK_BIT_OFFSET #undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE } -static ssize_t interrupt_sysfs_show( - struct device *device, struct device_attribute *attr, char *buf) +static ssize_t interrupt_sysfs_show(struct device *device, + struct device_attribute *attr, char *buf) { int i, ret; ssize_t written = 0, total_written = 0; @@ -318,22 +314,22 @@ static ssize_t interrupt_sysfs_show( } static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = { - GASKET_SYSFS_RO( - interrupt_counts, interrupt_sysfs_show, ATTR_INTERRUPT_COUNTS), + GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show, + ATTR_INTERRUPT_COUNTS), GASKET_END_OF_ATTR_ARRAY, }; -int gasket_interrupt_init( - struct gasket_dev *gasket_dev, const char *name, int type, - const struct gasket_interrupt_desc *interrupts, - int num_interrupts, int pack_width, int bar_index, - const struct gasket_wire_interrupt_offsets *wire_int_offsets) +int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, + int type, + const struct gasket_interrupt_desc *interrupts, + int num_interrupts, int pack_width, int bar_index, + const struct gasket_wire_interrupt_offsets *wire_int_offsets) { int ret; struct gasket_interrupt_data *interrupt_data; - interrupt_data = kzalloc( - sizeof(struct gasket_interrupt_data), GFP_KERNEL); + interrupt_data = kzalloc(sizeof(struct gasket_interrupt_data), + GFP_KERNEL); if (!interrupt_data) return -ENOMEM; gasket_dev->interrupt_data = interrupt_data; @@ -402,14 +398,14 @@ int gasket_interrupt_init( } gasket_interrupt_setup(gasket_dev); - gasket_sysfs_create_entries( - gasket_dev->dev_info.device, interrupt_sysfs_attrs); + gasket_sysfs_create_entries(gasket_dev->dev_info.device, + interrupt_sysfs_attrs); return 0; } -static void gasket_interrupt_msix_cleanup( - struct gasket_interrupt_data *interrupt_data) +static void +gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) { int i; @@ -528,9 +524,8 @@ int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) return GASKET_STATUS_ALIVE; } -int gasket_interrupt_set_eventfd( - struct gasket_interrupt_data *interrupt_data, int interrupt, - int event_fd) +int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data, + int interrupt, int event_fd) { struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd); @@ -544,8 +539,8 @@ int gasket_interrupt_set_eventfd( return 0; } -int gasket_interrupt_clear_eventfd( - struct gasket_interrupt_data *interrupt_data, int interrupt) +int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data, + int interrupt) { if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 805fee6..835af43 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -43,11 +43,11 @@ struct gasket_interrupt_data; * are not possible to set up, but is otherwise OK; that device will report * status LAMED.) */ -int gasket_interrupt_init( - struct gasket_dev *gasket_dev, const char *name, int type, - const struct gasket_interrupt_desc *interrupts, - int num_interrupts, int pack_width, int bar_index, - const struct gasket_wire_interrupt_offsets *wire_int_offsets); +int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, + int type, + const struct gasket_interrupt_desc *interrupts, + int num_interrupts, int pack_width, int bar_index, + const struct gasket_wire_interrupt_offsets *wire_int_offsets); /* * Clean up a device's interrupt structure. @@ -87,9 +87,8 @@ int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev); * * Returns 0 on success, a negative error code otherwise. */ -int gasket_interrupt_set_eventfd( - struct gasket_interrupt_data *interrupt_data, int interrupt, - int event_fd); +int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data, + int interrupt, int event_fd); /* * Removes an interrupt-eventfd association. @@ -98,8 +97,8 @@ int gasket_interrupt_set_eventfd( * * Removes any eventfd associated with the specified interrupt, if any. */ -int gasket_interrupt_clear_eventfd( - struct gasket_interrupt_data *interrupt_data, int interrupt); +int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data, + int interrupt); /* * The below functions exist for backwards compatibility. From e16116c477fdcd70a6002486f6f6616c9bc4f1c4 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:44 -0700 Subject: [PATCH 115/209] staging: gasket: TODO: remove entry for multi-line alignment style Multi-line alignment formatting issues fixed, remove the TODO entry for this. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- TODO | 3 --- 1 file changed, 3 deletions(-) diff --git a/TODO b/TODO index 6aa2a7f..6ff8e01 100644 --- a/TODO +++ b/TODO @@ -4,9 +4,6 @@ staging directory. - Use misc interface instead of major number for driver version description. - Add descriptions of module_param's - apex_get_status() should actually check status. -- Fix multi-line alignment formatting to look like: - int ret = long_function_name(device, VARIABLE1, VARIABLE2, - VARIABLE3, VARIABLE4); - "drivers" should never be dealing with "raw" sysfs calls or mess around with kobjects at all. The driver core should handle all of this for you automaically. There should not be a need for raw attribute macros. From 39f270999191ee60c3dca0b7c3e032e313bf4ab5 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:45 -0700 Subject: [PATCH 116/209] staging: gasket: apex: move driver-private defines out of apex.h apex.h is supposed to contain kernel-userspace interface definitions, but has a number of defines that are only used by apex_driver.c or are not used at all. Move driver implementation defines not shared with userspace to the driver source. Remove unused defines. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex.h | 62 ++------------------------------------------------- apex_driver.c | 29 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 60 deletions(-) diff --git a/apex.h b/apex.h index c7dd71e..5537d96 100644 --- a/apex.h +++ b/apex.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Apex kernel-userspace interface definition(s). + * Apex kernel-userspace interface definitions. * * Copyright (C) 2018 Google, Inc. */ @@ -8,66 +8,8 @@ #define __APEX_H__ #include -#include -#include "gasket.h" - -/* Structural definitions/macros. */ -/* The number of PCI BARs. */ -#define APEX_NUM_BARS 3 - -/* Size of a memory page in bytes, and the related number of bits to shift. */ -#define APEX_PAGE_SHIFT 12 -#define APEX_PAGE_SIZE BIT(APEX_PAGE_SHIFT) - -#define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */ - -/* - * Addresses are 2^3=8 bytes each. Page in second level page table holds - * APEX_PAGE_SIZE/8 addresses. - */ -#define APEX_ADDR_SHIFT 3 -#define APEX_LEVEL_SHIFT (APEX_PAGE_SHIFT - APEX_ADDR_SHIFT) -#define APEX_LEVEL_SIZE BIT(APEX_LEVEL_SHIFT) - -#define APEX_PAGE_TABLE_MAX 65536 -#define APEX_SIMPLE_PAGE_MAX APEX_PAGE_TABLE_MAX -#define APEX_EXTENDED_PAGE_MAX (APEX_PAGE_TABLE_MAX << APEX_LEVEL_SHIFT) - -/* Check reset 120 times */ -#define APEX_RESET_RETRY 120 -/* Wait 100 ms between checks. Total 12 sec wait maximum. */ -#define APEX_RESET_DELAY 100 - -#define APEX_CHIP_INIT_DONE 2 -#define APEX_RESET_ACCEPTED 0 - -enum apex_reset_types { - APEX_CHIP_REINIT_RESET = 3, -}; - -/* Interrupt defines */ -/* Gasket device interrupts enums must be dense (i.e., no empty slots). */ -enum apex_interrupt { - APEX_INTERRUPT_INSTR_QUEUE = 0, - APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1, - APEX_INTERRUPT_PARAM_QUEUE = 2, - APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3, - APEX_INTERRUPT_SC_HOST_0 = 4, - APEX_INTERRUPT_SC_HOST_1 = 5, - APEX_INTERRUPT_SC_HOST_2 = 6, - APEX_INTERRUPT_SC_HOST_3 = 7, - APEX_INTERRUPT_TOP_LEVEL_0 = 8, - APEX_INTERRUPT_TOP_LEVEL_1 = 9, - APEX_INTERRUPT_TOP_LEVEL_2 = 10, - APEX_INTERRUPT_TOP_LEVEL_3 = 11, - APEX_INTERRUPT_FATAL_ERR = 12, - APEX_INTERRUPT_COUNT = 13, -}; - -/* - * Clock Gating ioctl. - */ +/* Clock Gating ioctl. */ struct apex_gate_clock_ioctl { /* Enter or leave clock gated state. */ u64 enable; diff --git a/apex_driver.c b/apex_driver.c index 7ef0cb4..9b3cb8d 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -54,6 +54,17 @@ */ #define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192 +#define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */ + +enum apex_reset_types { + APEX_CHIP_REINIT_RESET = 3, +}; + +/* Check reset 120 times */ +#define APEX_RESET_RETRY 120 +/* Wait 100 ms between checks. Total 12 sec wait maximum. */ +#define APEX_RESET_DELAY 100 + /* Enumeration of the supported sysfs entries. */ enum sysfs_attribute_type { ATTR_KERNEL_HIB_PAGE_TABLE_SIZE, @@ -133,6 +144,24 @@ static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = { static const struct gasket_mappable_region cm_mappable_regions[1] = { { 0x0, APEX_CH_MEM_BYTES } }; +/* Gasket device interrupts enums must be dense (i.e., no empty slots). */ +enum apex_interrupt { + APEX_INTERRUPT_INSTR_QUEUE = 0, + APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1, + APEX_INTERRUPT_PARAM_QUEUE = 2, + APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3, + APEX_INTERRUPT_SC_HOST_0 = 4, + APEX_INTERRUPT_SC_HOST_1 = 5, + APEX_INTERRUPT_SC_HOST_2 = 6, + APEX_INTERRUPT_SC_HOST_3 = 7, + APEX_INTERRUPT_TOP_LEVEL_0 = 8, + APEX_INTERRUPT_TOP_LEVEL_1 = 9, + APEX_INTERRUPT_TOP_LEVEL_2 = 10, + APEX_INTERRUPT_TOP_LEVEL_3 = 11, + APEX_INTERRUPT_FATAL_ERR = 12, + APEX_INTERRUPT_COUNT = 13, +}; + /* Interrupt descriptors for Apex */ static struct gasket_interrupt_desc apex_interrupts[] = { { From 75e804acfbc86cf6d5634d6c271fd56ae55d2e2a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:46 -0700 Subject: [PATCH 117/209] staging: gasket: core: use bool type for ns_capable result When gasket core was converted from using capable() to use ns_capable() instead, the type of the variable holding the result should have been converted from int to bool. Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 4434452..f76f4a0 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1191,7 +1191,7 @@ static int gasket_open(struct inode *inode, struct file *filp) struct gasket_cdev_info *dev_info = container_of(inode->i_cdev, struct gasket_cdev_info, cdev); struct pid_namespace *pid_ns = task_active_pid_ns(current); - int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); + bool is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; @@ -1270,7 +1270,7 @@ static int gasket_release(struct inode *inode, struct file *file) struct gasket_cdev_info *dev_info = container_of(inode->i_cdev, struct gasket_cdev_info, cdev); struct pid_namespace *pid_ns = task_active_pid_ns(current); - int is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); + bool is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN); gasket_dev = dev_info->gasket_dev_ptr; driver_desc = gasket_dev->internal_desc->driver_desc; From fe8754893579cfeb5ad25859762e8d76ad5bb3c6 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:47 -0700 Subject: [PATCH 118/209] Revert "staging: gasket: page table: hold references to device and pci_dev" gasket_free_dev() is called only from driver PCI probe and remove function. It is guaranteed that that pci_dev structure is not going anywhere during that time; there is no need to take this additional reference. This reverts commit dd9d1502feea3c23d412f289aad79e1d4e86d45d. Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 13e1d09..ed6ab3c 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -280,7 +280,7 @@ int gasket_page_table_init(struct gasket_page_table **ppg_tbl, pg_tbl->extended_offset_reg = (u64 __iomem *)&bar_data->virt_base[page_table_config->extended_reg]; pg_tbl->device = get_device(device); - pg_tbl->pci_dev = pci_dev_get(pci_dev); + pg_tbl->pci_dev = pci_dev; dev_dbg(device, "Page table initialized successfully\n"); @@ -378,7 +378,6 @@ void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl) pg_tbl->entries = NULL; put_device(pg_tbl->device); - pci_dev_put(pg_tbl->pci_dev); kfree(pg_tbl); } From 0d1413c4e174e3e50ef0d98daf599bca9acc98f1 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 31 Jul 2018 13:24:48 -0700 Subject: [PATCH 119/209] staging: gasket: page table: fix header file include guard symbol The include guard symbol for gasket_page_table.h is out-of-date. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gasket_page_table.h b/gasket_page_table.h index 00c06f0..7b01b73 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -9,8 +9,8 @@ * Copyright (C) 2018 Google, Inc. */ -#ifndef __GASKET_ADDR_TRNSL_H__ -#define __GASKET_ADDR_TRNSL_H__ +#ifndef __GASKET_PAGE_TABLE_H__ +#define __GASKET_PAGE_TABLE_H__ #include #include @@ -246,4 +246,4 @@ void gasket_free_coherent_memory_all(struct gasket_dev *gasket_dev, int gasket_set_user_virt(struct gasket_dev *gasket_dev, uint64_t size, dma_addr_t dma_address, ulong vma); -#endif +#endif /* __GASKET_PAGE_TABLE_H__ */ From ca7d6796aaa4d32fa9ea2dd2d317da3cf0ddf950 Mon Sep 17 00:00:00 2001 From: Ivan Bornyakov Date: Wed, 1 Aug 2018 21:37:44 +0300 Subject: [PATCH 120/209] staging: gasket: sysfs: fix potential null dereference Add handling of possible allocation failure. Reported by smatch: drivers/staging/gasket/gasket_sysfs.c:105 put_mapping() error: potential null dereference 'files_to_remove'. (kcalloc returns null) Signed-off-by: Ivan Bornyakov Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index a4bfca4..56d62ae 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -101,6 +101,11 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) files_to_remove = kcalloc(num_files_to_remove, sizeof(*files_to_remove), GFP_KERNEL); + if (!files_to_remove) { + mutex_unlock(&mapping->mutex); + return; + } + for (i = 0; i < num_files_to_remove; i++) files_to_remove[i] = mapping->attributes[i].attr; From 28e76c9523f4d473285c8679268562274a9c2f2d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 2 Aug 2018 01:42:43 -0700 Subject: [PATCH 121/209] staging: gasket: remove "reset type" param from framework The "type of reset" parameter to the gasket device reset APIs isn't required by the only gasket device submitted upstream, apex. The framework documents the param as private to the device driver and a pass-through at the gasket layer, but the gasket core calls the device driver with a hardcoded reset type of zero, which is not documented as having a predefined meaning. In light of all this, remove the reset type parameter from the framework. Remove the reset ioctl reset type parameter, and bump the framework version number to reflect the interface change. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket.h | 4 ++-- gasket_constants.h | 2 +- gasket_core.c | 11 +++++------ gasket_core.h | 13 +++---------- gasket_ioctl.c | 3 +-- 5 files changed, 12 insertions(+), 21 deletions(-) diff --git a/gasket.h b/gasket.h index 9f709f0..a0f065c 100644 --- a/gasket.h +++ b/gasket.h @@ -52,8 +52,8 @@ struct gasket_coherent_alloc_config_ioctl { /* Base number for all Gasket-common IOCTLs */ #define GASKET_IOCTL_BASE 0xDC -/* Reset the device using the specified reset type. */ -#define GASKET_IOCTL_RESET _IOW(GASKET_IOCTL_BASE, 0, unsigned long) +/* Reset the device. */ +#define GASKET_IOCTL_RESET _IO(GASKET_IOCTL_BASE, 0) /* Associate the specified [event]fd with the specified interrupt. */ #define GASKET_IOCTL_SET_EVENTFD \ diff --git a/gasket_constants.h b/gasket_constants.h index 82ed3f2..50d87c7 100644 --- a/gasket_constants.h +++ b/gasket_constants.h @@ -3,7 +3,7 @@ #ifndef __GASKET_CONSTANTS_H__ #define __GASKET_CONSTANTS_H__ -#define GASKET_FRAMEWORK_VERSION "1.1.1" +#define GASKET_FRAMEWORK_VERSION "1.1.2" /* * The maximum number of simultaneous device types supported by the framework. diff --git a/gasket_core.c b/gasket_core.c index f76f4a0..2b75f10 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1294,7 +1294,7 @@ static int gasket_release(struct inode *inode, struct file *file) ownership->owner = 0; /* Forces chip reset before we unmap the page tables. */ - driver_desc->device_reset_cb(gasket_dev, 0); + driver_desc->device_reset_cb(gasket_dev); for (i = 0; i < driver_desc->num_page_tables; ++i) { gasket_page_table_unmap_all(gasket_dev->page_table[i]); @@ -1622,18 +1622,18 @@ const char *gasket_num_name_lookup(uint num, } EXPORT_SYMBOL(gasket_num_name_lookup); -int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type) +int gasket_reset(struct gasket_dev *gasket_dev) { int ret; mutex_lock(&gasket_dev->mutex); - ret = gasket_reset_nolock(gasket_dev, reset_type); + ret = gasket_reset_nolock(gasket_dev); mutex_unlock(&gasket_dev->mutex); return ret; } EXPORT_SYMBOL(gasket_reset); -int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) +int gasket_reset_nolock(struct gasket_dev *gasket_dev) { int ret; int i; @@ -1643,8 +1643,7 @@ int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type) if (!driver_desc->device_reset_cb) return 0; - /* Perform a device reset of the requested type. */ - ret = driver_desc->device_reset_cb(gasket_dev, reset_type); + ret = driver_desc->device_reset_cb(gasket_dev); if (ret) { dev_dbg(gasket_dev->dev, "Device reset cb returned %d.\n", ret); diff --git a/gasket_core.h b/gasket_core.h index 713bf42..67f5960 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -580,17 +580,12 @@ struct gasket_driver_desc { /* * device_reset_cb: Reset the hardware in question. * @dev: Pointer to the gasket_dev structure for this device. - * @type: Integer representing reset type. (All - * Gasket resets have an integer representing their type - * defined in (device)_ioctl.h; the specific resets are - * device-dependent, but are handled in the device-specific - * callback anyways.) * * Called by reset ioctls. This function should not * lock the gasket_dev mutex. It should return 0 on success * and an error on failure. */ - int (*device_reset_cb)(struct gasket_dev *dev, uint reset_type); + int (*device_reset_cb)(struct gasket_dev *dev); }; /* @@ -615,15 +610,13 @@ void gasket_unregister_device(const struct gasket_driver_desc *desc); /* * Reset the Gasket device. * @gasket_dev: Gasket device struct. - * @reset_type: Uint representing requested reset type. Should be - * valid in the underlying callback. * * Calls device_reset_cb. Returns 0 on success and an error code othewrise. * gasket_reset_nolock will not lock the mutex, gasket_reset will. * */ -int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type); -int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type); +int gasket_reset(struct gasket_dev *gasket_dev); +int gasket_reset_nolock(struct gasket_dev *gasket_dev); /* * Memory management functions. These will likely be spun off into their own diff --git a/gasket_ioctl.c b/gasket_ioctl.c index d3397cc..0ca48e6 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -304,8 +304,7 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) */ switch (cmd) { case GASKET_IOCTL_RESET: - trace_gasket_ioctl_integer_data(arg); - retval = gasket_reset(gasket_dev, arg); + retval = gasket_reset(gasket_dev); break; case GASKET_IOCTL_SET_EVENTFD: retval = gasket_set_event_fd(gasket_dev, argp); From 7ec73c69e12040d476b16f714f706dd44f5c3784 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 2 Aug 2018 01:42:44 -0700 Subject: [PATCH 122/209] staging: gasket: apex: drop reset type param Apex doesn't implement different types of resets based on the reset type param passed through the gasket layer or from userspace via the gasket_reset ioctl. The reset type is dropped from the gasket framework in a previous patch due to a lack of present need and non-conforming use of this parameter by the framework. Drop the parameter from the apex driver as well. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 9b3cb8d..953bdb1 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -56,10 +56,6 @@ #define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */ -enum apex_reset_types { - APEX_CHIP_REINIT_RESET = 3, -}; - /* Check reset 120 times */ #define APEX_RESET_RETRY 120 /* Wait 100 ms between checks. Total 12 sec wait maximum. */ @@ -258,7 +254,7 @@ static int apex_get_status(struct gasket_dev *gasket_dev) } /* Enter GCB reset state. */ -static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) +static int apex_enter_reset(struct gasket_dev *gasket_dev) { if (bypass_top_level) return 0; @@ -313,7 +309,7 @@ static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type) } /* Quit GCB reset state. */ -static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type) +static int apex_quit_reset(struct gasket_dev *gasket_dev) { u32 val0, val1; @@ -413,7 +409,7 @@ static int apex_device_cleanup(struct gasket_dev *gasket_dev) __func__, gasket_dev, hib_error, scalar_error); if (allow_power_save) - ret = apex_enter_reset(gasket_dev, APEX_CHIP_REINIT_RESET); + ret = apex_enter_reset(gasket_dev); return ret; } @@ -429,7 +425,7 @@ static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) } /* Reset the hardware, then quit reset. Called on device open. */ -static int apex_reset(struct gasket_dev *gasket_dev, uint type) +static int apex_reset(struct gasket_dev *gasket_dev) { int ret; @@ -442,11 +438,11 @@ static int apex_reset(struct gasket_dev *gasket_dev, uint type) */ dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__); - ret = apex_enter_reset(gasket_dev, type); + ret = apex_enter_reset(gasket_dev); if (ret) return ret; } - ret = apex_quit_reset(gasket_dev, type); + ret = apex_quit_reset(gasket_dev); return ret; } @@ -456,7 +452,7 @@ static int apex_add_dev_cb(struct gasket_dev *gasket_dev) ulong page_table_ready, msix_table_ready; int retries = 0; - apex_reset(gasket_dev, 0); + apex_reset(gasket_dev); while (retries < APEX_RESET_RETRY) { page_table_ready = @@ -675,7 +671,7 @@ static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) /* On device open, perform a core reinit reset. */ static int apex_device_open_cb(struct gasket_dev *gasket_dev) { - return gasket_reset_nolock(gasket_dev, APEX_CHIP_REINIT_RESET); + return gasket_reset_nolock(gasket_dev); } static const struct pci_device_id apex_pci_ids[] = { From f91b2869523120e0cf8ce94a930f5fbf166e074a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 2 Aug 2018 18:49:49 -0700 Subject: [PATCH 123/209] staging: gasket: core: remove registration logs Remove logs for loading gasket drivers. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 2b75f10..fa477d0 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1758,9 +1758,6 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) } mutex_unlock(&g_mutex); - pr_info("Loaded %s driver, framework version %s\n", - driver_desc->name, GASKET_FRAMEWORK_VERSION); - if (desc_idx == -1) { pr_err("Too many Gasket drivers loaded: %d\n", GASKET_FRAMEWORK_DESC_MAX); @@ -1808,7 +1805,6 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) goto fail2; } - pr_info("Driver registered successfully.\n"); return 0; fail2: From 82374d85bd301d9a1a0823b10da7bbfe438d42be Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 2 Aug 2018 18:49:50 -0700 Subject: [PATCH 124/209] staging: gasket: core: device register debug log cleanups At device/driver registration time, convert a not-very-informative info message to a more informative debug message, drop some not overly helpful debug messages. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index fa477d0..91db71c 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1735,7 +1735,8 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) int desc_idx = -1; struct gasket_internal_desc *internal; - pr_info("Initializing Gasket framework device\n"); + pr_debug("Loading %s driver version %s\n", driver_desc->name, + driver_desc->driver_version); /* Check for duplicates and find a free slot. */ mutex_lock(&g_mutex); @@ -1764,8 +1765,6 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) return -EBUSY; } - /* Internal structure setup. */ - pr_debug("Performing initial internal structure setup.\n"); internal = &g_descs[desc_idx]; mutex_init(&internal->mutex); memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX); @@ -1788,7 +1787,6 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) * Not using pci_register_driver() (without underscores), as it * depends on KBUILD_MODNAME, and this is a shared file. */ - pr_debug("Registering PCI driver.\n"); ret = __pci_register_driver(&internal->pci, driver_desc->module, driver_desc->name); if (ret) { @@ -1796,7 +1794,6 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) goto fail1; } - pr_debug("Registering char driver.\n"); ret = register_chrdev_region(MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, driver_desc->name); From 0f3eba23745c6474720cd25089315e383a0cc856 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 2 Aug 2018 18:49:51 -0700 Subject: [PATCH 125/209] staging: gasket: core: add subsystem and device info to logs Identify gasket as the subsystem printing various messages. Add the driver name to appropriate messages to indicate which driver has a problem. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 91db71c..93a4d9f 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -6,6 +6,9 @@ * * Copyright (C) 2018 Google, Inc. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include "gasket_core.h" #include "gasket_interrupt.h" @@ -208,7 +211,7 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL); if (!gasket_dev) { - pr_err("no memory for device\n"); + pr_err("no memory for device %s\n", kobj_name); return -ENOMEM; } internal_desc->devs[dev_idx] = gasket_dev; @@ -1760,7 +1763,7 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) mutex_unlock(&g_mutex); if (desc_idx == -1) { - pr_err("Too many Gasket drivers loaded: %d\n", + pr_err("too many drivers loaded, max %d\n", GASKET_FRAMEWORK_DESC_MAX); return -EBUSY; } @@ -1790,7 +1793,8 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) ret = __pci_register_driver(&internal->pci, driver_desc->module, driver_desc->name); if (ret) { - pr_err("cannot register pci driver [ret=%d]\n", ret); + pr_err("cannot register %s pci driver [ret=%d]\n", + driver_desc->name, ret); goto fail1; } @@ -1798,7 +1802,8 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) driver_desc->minor), GASKET_DEV_MAX, driver_desc->name); if (ret) { - pr_err("cannot register char driver [ret=%d]\n", ret); + pr_err("cannot register %s char driver [ret=%d]\n", + driver_desc->name, ret); goto fail2; } From fa8e12941eae309d84cd5623ace9150545447ec8 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 2 Aug 2018 18:49:52 -0700 Subject: [PATCH 126/209] Revert "staging: gasket: core: hold reference to pci_dev while used" There's no need to take an additional reference on the pci_dev structure for the pointer copy saved in gasket data structures. This reverts commit: 8dd8a48b9a7d ("staging: gasket: core: hold reference to pci_dev while used") Reported-by: Dmitry Torokhov Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 93a4d9f..2d209e3 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -255,7 +255,6 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) internal_desc->devs[gasket_dev->dev_idx] = NULL; mutex_unlock(&internal_desc->mutex); put_device(gasket_dev->dev); - pci_dev_put(gasket_dev->pci_dev); kfree(gasket_dev); } @@ -1477,7 +1476,7 @@ static int gasket_pci_probe(struct pci_dev *pci_dev, ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); if (ret) return ret; - gasket_dev->pci_dev = pci_dev_get(pci_dev); + gasket_dev->pci_dev = pci_dev; if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { pr_err("Cannot create %s device %s [ret = %ld]\n", driver_desc->name, gasket_dev->dev_info.name, From febda1fad456a7ada1a2af17b73b201f89313f61 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:35 -0700 Subject: [PATCH 127/209] staging: gasket: sysfs: clean up state if ENOMEM removing mapping If kcalloc() returns NULL in put_mapping(), continue to clean up state, including dropping the reference on the struct device and free attribute memory. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_sysfs.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/gasket_sysfs.c b/gasket_sysfs.c index 56d62ae..fc45f0d 100644 --- a/gasket_sysfs.c +++ b/gasket_sysfs.c @@ -101,13 +101,12 @@ static void put_mapping(struct gasket_sysfs_mapping *mapping) files_to_remove = kcalloc(num_files_to_remove, sizeof(*files_to_remove), GFP_KERNEL); - if (!files_to_remove) { - mutex_unlock(&mapping->mutex); - return; - } - - for (i = 0; i < num_files_to_remove; i++) - files_to_remove[i] = mapping->attributes[i].attr; + if (files_to_remove) + for (i = 0; i < num_files_to_remove; i++) + files_to_remove[i] = + mapping->attributes[i].attr; + else + num_files_to_remove = 0; kfree(mapping->attributes); mapping->attributes = NULL; From c9f462b11e2d410daf0a26b45f8bba4b03614615 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:36 -0700 Subject: [PATCH 128/209] staging: gasket: core: move core PCI calls to device drivers Remove gasket wrapping of PCI probe, enable, disable, and remove functions. Replace with calls to add and remove PCI gasket devices, to be called by the gasket device drivers. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 82 ++++++++++++++------------------------------------- gasket_core.h | 6 ++++ 2 files changed, 28 insertions(+), 60 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 2d209e3..01cafe1 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -51,9 +51,6 @@ struct gasket_internal_desc { /* Kernel-internal device class. */ struct class *class; - /* PCI subsystem metadata associated with this driver. */ - struct pci_driver pci; - /* Instantiated / present devices of this type. */ struct gasket_dev *devs[GASKET_DEV_MAX]; }; @@ -368,10 +365,10 @@ static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num) } /* - * Setup PCI & set up memory mapping for the specified device. + * Setup PCI memory mapping for the specified device. * - * Enables the PCI device, reads the BAR registers and sets up pointers to the - * device's memory mapped IO space. + * Reads the BAR registers and sets up pointers to the device's memory mapped + * IO space. * * Returns 0 on success and a negative value otherwise. */ @@ -380,14 +377,6 @@ static int gasket_setup_pci(struct pci_dev *pci_dev, { int i, mapped_bars, ret; - ret = pci_enable_device(pci_dev); - if (ret) { - dev_err(gasket_dev->dev, "cannot enable PCI device\n"); - return ret; - } - - pci_set_master(pci_dev); - for (i = 0; i < GASKET_NUM_BARS; i++) { ret = gasket_map_pci_bar(gasket_dev, i); if (ret) { @@ -402,19 +391,16 @@ fail: for (i = 0; i < mapped_bars; i++) gasket_unmap_pci_bar(gasket_dev, i); - pci_disable_device(pci_dev); return -ENOMEM; } -/* Unmaps memory and cleans up PCI for the specified device. */ +/* Unmaps memory for the specified device. */ static void gasket_cleanup_pci(struct gasket_dev *gasket_dev) { int i; for (i = 0; i < GASKET_NUM_BARS; i++) gasket_unmap_pci_bar(gasket_dev, i); - - pci_disable_device(gasket_dev->pci_dev); } /* Determine the health of the Gasket device. */ @@ -1443,15 +1429,14 @@ static int gasket_enable_dev(struct gasket_internal_desc *internal_desc, } /* - * PCI subsystem probe function. + * Add PCI gasket device. * - * Called when a Gasket device is found. Allocates device metadata, maps device - * memory, and calls gasket_enable_dev to prepare the device for active use. - * - * Returns 0 if successful and a negative value otherwise. + * Called by Gasket device probe function. + * Allocates device metadata, maps device memory, and calls gasket_enable_dev + * to prepare the device for active use. */ -static int gasket_pci_probe(struct pci_dev *pci_dev, - const struct pci_device_id *id) +int gasket_pci_add_device(struct pci_dev *pci_dev, + struct gasket_dev **gasket_devp) { int ret; const char *kobj_name = dev_name(&pci_dev->dev); @@ -1460,13 +1445,14 @@ static int gasket_pci_probe(struct pci_dev *pci_dev, const struct gasket_driver_desc *driver_desc; struct device *parent; - pr_info("Add Gasket device %s\n", kobj_name); + pr_debug("add PCI device %s\n", kobj_name); mutex_lock(&g_mutex); internal_desc = lookup_internal_desc(pci_dev); mutex_unlock(&g_mutex); if (!internal_desc) { - pr_err("PCI probe called for unknown driver type\n"); + dev_err(&pci_dev->dev, + "PCI add device called for unknown driver type\n"); return -ENODEV; } @@ -1530,6 +1516,7 @@ static int gasket_pci_probe(struct pci_dev *pci_dev, goto fail5; } + *gasket_devp = gasket_dev; return 0; fail5: @@ -1545,14 +1532,10 @@ fail1: gasket_free_dev(gasket_dev); return ret; } +EXPORT_SYMBOL(gasket_pci_add_device); -/* - * PCI subsystem remove function. - * - * Called to remove a Gasket device. Finds the device in the device list and - * cleans up metadata. - */ -static void gasket_pci_remove(struct pci_dev *pci_dev) +/* Remove a PCI gasket device. */ +void gasket_pci_remove_device(struct pci_dev *pci_dev) { int i; struct gasket_internal_desc *internal_desc; @@ -1583,8 +1566,8 @@ static void gasket_pci_remove(struct pci_dev *pci_dev) if (!gasket_dev) return; - pr_info("remove %s device %s\n", internal_desc->driver_desc->name, - gasket_dev->kobj_name); + dev_dbg(gasket_dev->dev, "remove %s PCI gasket device\n", + internal_desc->driver_desc->name); gasket_disable_dev(gasket_dev); gasket_cleanup_pci(gasket_dev); @@ -1597,6 +1580,7 @@ static void gasket_pci_remove(struct pci_dev *pci_dev) device_destroy(internal_desc->class, gasket_dev->dev_info.devt); gasket_free_dev(gasket_dev); } +EXPORT_SYMBOL(gasket_pci_remove_device); /** * Lookup a name by number in a num_name table. @@ -1770,11 +1754,6 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) internal = &g_descs[desc_idx]; mutex_init(&internal->mutex); memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX); - memset(&internal->pci, 0, sizeof(internal->pci)); - internal->pci.name = driver_desc->name; - internal->pci.id_table = driver_desc->pci_id_table; - internal->pci.probe = gasket_pci_probe; - internal->pci.remove = gasket_pci_remove; internal->class = class_create(driver_desc->module, driver_desc->name); @@ -1785,33 +1764,18 @@ int gasket_register_device(const struct gasket_driver_desc *driver_desc) goto unregister_gasket_driver; } - /* - * Not using pci_register_driver() (without underscores), as it - * depends on KBUILD_MODNAME, and this is a shared file. - */ - ret = __pci_register_driver(&internal->pci, driver_desc->module, - driver_desc->name); - if (ret) { - pr_err("cannot register %s pci driver [ret=%d]\n", - driver_desc->name, ret); - goto fail1; - } - ret = register_chrdev_region(MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX, driver_desc->name); if (ret) { pr_err("cannot register %s char driver [ret=%d]\n", driver_desc->name, ret); - goto fail2; + goto destroy_class; } return 0; -fail2: - pci_unregister_driver(&internal->pci); - -fail1: +destroy_class: class_destroy(internal->class); unregister_gasket_driver: @@ -1848,8 +1812,6 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) unregister_chrdev_region(MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX); - pci_unregister_driver(&internal_desc->pci); - class_destroy(internal_desc->class); /* Finally, effectively "remove" the driver. */ diff --git a/gasket_core.h b/gasket_core.h index 67f5960..9f9bc66 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -607,6 +607,12 @@ int gasket_register_device(const struct gasket_driver_desc *desc); */ void gasket_unregister_device(const struct gasket_driver_desc *desc); +/* Add a PCI gasket device. */ +int gasket_pci_add_device(struct pci_dev *pci_dev, + struct gasket_dev **gasket_devp); +/* Remove a PCI gasket device. */ +void gasket_pci_remove_device(struct pci_dev *pci_dev); + /* * Reset the Gasket device. * @gasket_dev: Gasket device struct. From ea7e508c0d797b55313375a84d689660492370e1 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:37 -0700 Subject: [PATCH 129/209] staging: gasket: apex: move PCI core calls to apex driver Apex driver moves PCI core calls like probe, enable, and remove from gasket to apex. Call new functions in gasket to register apex as a PCI device to the gasket framework. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/apex_driver.c b/apex_driver.c index 953bdb1..fe9399e 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -685,6 +686,36 @@ static void apex_pci_fixup_class(struct pci_dev *pdev) DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); +static int apex_pci_probe(struct pci_dev *pci_dev, + const struct pci_device_id *id) +{ + int ret; + struct gasket_dev *gasket_dev; + + ret = pci_enable_device(pci_dev); + if (ret) { + dev_err(&pci_dev->dev, "error enabling PCI device\n"); + return ret; + } + + pci_set_master(pci_dev); + + ret = gasket_pci_add_device(pci_dev, &gasket_dev); + if (ret) { + dev_err(&pci_dev->dev, "error adding gasket device\n"); + pci_disable_device(pci_dev); + return ret; + } + + return 0; +} + +static void apex_pci_remove(struct pci_dev *pci_dev) +{ + gasket_pci_remove_device(pci_dev); + pci_disable_device(pci_dev); +} + static struct gasket_driver_desc apex_desc = { .name = "apex", .driver_version = APEX_DRIVER_VERSION, @@ -736,13 +767,29 @@ static struct gasket_driver_desc apex_desc = { .device_reset_cb = apex_reset, }; +static struct pci_driver apex_pci_driver = { + .name = "apex", + .probe = apex_pci_probe, + .remove = apex_pci_remove, + .id_table = apex_pci_ids, +}; + static int __init apex_init(void) { - return gasket_register_device(&apex_desc); + int ret; + + ret = gasket_register_device(&apex_desc); + if (ret) + return ret; + ret = pci_register_driver(&apex_pci_driver); + if (ret) + gasket_unregister_device(&apex_desc); + return ret; } static void apex_exit(void) { + pci_unregister_driver(&apex_pci_driver); gasket_unregister_device(&apex_desc); } MODULE_DESCRIPTION("Google Apex driver"); From 94080ed1ab510efb947602500a2f50f9c60e193a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:38 -0700 Subject: [PATCH 130/209] staging: gasket: core: convert remaining info logs to debug Remaining info-level logs in gasket core converted to debug-level; the information is not needed during normal system operation. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 01cafe1..2741256 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1819,7 +1819,7 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) g_descs[desc_idx].driver_desc = NULL; mutex_unlock(&g_mutex); - pr_info("removed %s driver\n", driver_desc->name); + pr_debug("removed %s driver\n", driver_desc->name); } EXPORT_SYMBOL(gasket_unregister_device); @@ -1827,7 +1827,7 @@ static int __init gasket_init(void) { int i; - pr_info("Performing one-time init of the Gasket framework.\n"); + pr_debug("%s\n", __func__); /* Check for duplicates and find a free slot. */ mutex_lock(&g_mutex); for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { @@ -1843,8 +1843,7 @@ static int __init gasket_init(void) static void __exit gasket_exit(void) { - /* No deinit/dealloc needed at present. */ - pr_info("Removing Gasket framework module.\n"); + pr_debug("%s\n", __func__); } MODULE_DESCRIPTION("Google Gasket driver framework"); MODULE_VERSION(GASKET_FRAMEWORK_VERSION); From 68d0d4ac9db35460aff35da9b9239447ec8d1aa3 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:39 -0700 Subject: [PATCH 131/209] staging: gasket: core: remove device enable and disable callbacks Device enable/disable operations are moving from being initiated through the gasket framework to being initiated by the gasket device driver. The driver can perform any processing needed for these operations before or after the calls into the framework. Neither of these callbacks are implemented for the only gasket driver upstream today, apex. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 9 --------- gasket_core.h | 27 ++------------------------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 2741256..b070efa 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -648,8 +648,6 @@ static void gasket_disable_dev(struct gasket_dev *gasket_dev) gasket_page_table_cleanup(gasket_dev->page_table[i]); } } - - check_and_invoke_callback(gasket_dev, driver_desc->disable_dev_cb); } /* @@ -1408,13 +1406,6 @@ static int gasket_enable_dev(struct gasket_internal_desc *internal_desc, } gasket_dev->hardware_revision = ret; - ret = check_and_invoke_callback(gasket_dev, driver_desc->enable_dev_cb); - if (ret) { - dev_err(gasket_dev->dev, "Error in enable device cb: %d\n", - ret); - return ret; - } - /* device_status_cb returns a device status, not an error code. */ gasket_dev->status = gasket_get_hw_status(gasket_dev); if (gasket_dev->status == GASKET_STATUS_DEAD) diff --git a/gasket_core.h b/gasket_core.h index 9f9bc66..5d40bc7 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -473,34 +473,11 @@ struct gasket_driver_desc { */ int (*device_close_cb)(struct gasket_dev *dev); - /* - * enable_dev_cb: Callback immediately before enabling the device. - * @dev: Pointer to the gasket_dev struct for this driver instance. - * - * This callback is invoked after the device has been added and all BAR - * spaces mapped, immediately before registering and enabling the - * [character] device via cdev_add. If this call fails (returns - * nonzero), disable_dev_cb will be called. - * - * Note that cdev are initialized but not active - * (cdev_add has not yet been called) when this callback is invoked. - */ - int (*enable_dev_cb)(struct gasket_dev *dev); - - /* - * disable_dev_cb: Callback immediately after disabling the device. - * @dev: Pointer to the gasket_dev struct for this driver instance. - * - * Called during device shutdown, immediately after disabling device - * operations via cdev_del. - */ - int (*disable_dev_cb)(struct gasket_dev *dev); - /* * sysfs_setup_cb: Callback to set up driver-specific sysfs nodes. * @dev: Pointer to the gasket_dev struct for this device. * - * Called just before enable_dev_cb. + * Called during the add gasket device call. * */ int (*sysfs_setup_cb)(struct gasket_dev *dev); @@ -509,7 +486,7 @@ struct gasket_driver_desc { * sysfs_cleanup_cb: Callback to clean up driver-specific sysfs nodes. * @dev: Pointer to the gasket_dev struct for this device. * - * Called just before disable_dev_cb. + * Called during device disable processing. * */ int (*sysfs_cleanup_cb)(struct gasket_dev *dev); From a1fc286eef17b7d34d22b146af7e44d01d24a987 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:40 -0700 Subject: [PATCH 132/209] staging: gasket: apex: remove device enable and disable callbacks These are not implemented for apex, and are now being removed from the gasket framework. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index fe9399e..9b13929 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -752,9 +752,6 @@ static struct gasket_driver_desc apex_desc = { .add_dev_cb = apex_add_dev_cb, .remove_dev_cb = NULL, - .enable_dev_cb = NULL, - .disable_dev_cb = NULL, - .sysfs_setup_cb = apex_sysfs_setup_cb, .sysfs_cleanup_cb = NULL, From 8c61b5327be0eb29cb1bce723ffbd7c32f121f17 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:41 -0700 Subject: [PATCH 133/209] staging: gasket: core: let device driver enable/disable gasket device Move gasket device enable/disable functions from internal calls to external calls from the gasket device drivers. The device driver will call these functions at appropriate times in its processing, placing the device driver in control of this sequence and reducing the need for callbacks from framework back to the device drivers during the enable/disable sequences. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 22 ++++++++-------------- gasket_core.h | 6 ++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index b070efa..fad4883 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -628,7 +628,7 @@ static int gasket_add_cdev(struct gasket_cdev_info *dev_info, } /* Disable device operations. */ -static void gasket_disable_dev(struct gasket_dev *gasket_dev) +void gasket_disable_device(struct gasket_dev *gasket_dev) { const struct gasket_driver_desc *driver_desc = gasket_dev->internal_desc->driver_desc; @@ -649,6 +649,7 @@ static void gasket_disable_dev(struct gasket_dev *gasket_dev) } } } +EXPORT_SYMBOL(gasket_disable_device); /* * Registered descriptor lookup. @@ -1350,13 +1351,12 @@ static const struct file_operations gasket_file_ops = { }; /* Perform final init and marks the device as active. */ -static int gasket_enable_dev(struct gasket_internal_desc *internal_desc, - struct gasket_dev *gasket_dev) +int gasket_enable_device(struct gasket_dev *gasket_dev) { int tbl_idx; int ret; const struct gasket_driver_desc *driver_desc = - internal_desc->driver_desc; + gasket_dev->internal_desc->driver_desc; ret = gasket_interrupt_init(gasket_dev, driver_desc->name, driver_desc->interrupt_type, @@ -1418,13 +1418,15 @@ static int gasket_enable_dev(struct gasket_internal_desc *internal_desc, return 0; } +EXPORT_SYMBOL(gasket_enable_device); /* * Add PCI gasket device. * * Called by Gasket device probe function. - * Allocates device metadata, maps device memory, and calls gasket_enable_dev - * to prepare the device for active use. + * Allocates device metadata and maps device memory. The device driver must + * call gasket_enable_device after driver init is complete to place the device + * in active use. */ int gasket_pci_add_device(struct pci_dev *pci_dev, struct gasket_dev **gasket_devp) @@ -1500,13 +1502,6 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, goto fail5; } - ret = gasket_enable_dev(internal_desc, gasket_dev); - if (ret) { - pr_err("cannot setup %s device\n", driver_desc->name); - gasket_disable_dev(gasket_dev); - goto fail5; - } - *gasket_devp = gasket_dev; return 0; @@ -1560,7 +1555,6 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) dev_dbg(gasket_dev->dev, "remove %s PCI gasket device\n", internal_desc->driver_desc->name); - gasket_disable_dev(gasket_dev); gasket_cleanup_pci(gasket_dev); check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); diff --git a/gasket_core.h b/gasket_core.h index 5d40bc7..9c143eb 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -590,6 +590,12 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, /* Remove a PCI gasket device. */ void gasket_pci_remove_device(struct pci_dev *pci_dev); +/* Enable a Gasket device. */ +int gasket_enable_device(struct gasket_dev *gasket_dev); + +/* Disable a Gasket device. */ +void gasket_disable_device(struct gasket_dev *gasket_dev); + /* * Reset the Gasket device. * @gasket_dev: Gasket device struct. From 1c34a962b394dceb77d11aef6401892415e8a687 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:42 -0700 Subject: [PATCH 134/209] staging: gasket: apex: enable/disable gasket device from apex Gasket framework now places device drivers in charge of calling APIs to enable and disable gasket device operations. Make the appropriate calls from the apex driver. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apex_driver.c b/apex_driver.c index 9b13929..a4b051c 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -707,11 +707,23 @@ static int apex_pci_probe(struct pci_dev *pci_dev, return ret; } + pci_set_drvdata(pci_dev, gasket_dev); + ret = gasket_enable_device(gasket_dev); + if (ret) { + dev_err(&pci_dev->dev, "error enabling gasket device\n"); + gasket_pci_remove_device(pci_dev); + pci_disable_device(pci_dev); + return ret; + } + return 0; } static void apex_pci_remove(struct pci_dev *pci_dev) { + struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev); + + gasket_disable_device(gasket_dev); gasket_pci_remove_device(pci_dev); pci_disable_device(pci_dev); } From a72c98190c5577a7272b16350d4f25e40c0f09cc Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:43 -0700 Subject: [PATCH 135/209] staging: gasket: core: delete device add and remove callbacks Gasket device drivers are now in charge of orchestrating the device add and removal sequences, so the callbacks from the framework to the device drivers for these events are no longer needed. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 10 ---------- gasket_core.h | 29 ----------------------------- 2 files changed, 39 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index fad4883..0d76e18 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1468,12 +1468,6 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, if (ret) goto fail2; - ret = check_and_invoke_callback(gasket_dev, driver_desc->add_dev_cb); - if (ret) { - dev_err(gasket_dev->dev, "Error in add device cb: %d\n", ret); - goto fail2; - } - ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device, gasket_dev); if (ret) @@ -1512,7 +1506,6 @@ fail3: gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); fail2: gasket_cleanup_pci(gasket_dev); - check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); device_destroy(internal_desc->class, gasket_dev->dev_info.devt); fail1: gasket_free_dev(gasket_dev); @@ -1559,9 +1552,6 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); - - check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb); - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); gasket_free_dev(gasket_dev); } diff --git a/gasket_core.h b/gasket_core.h index 9c143eb..0ef0a26 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -302,12 +302,6 @@ struct gasket_dev { /* Hardware revision value for this device. */ int hardware_revision; - /* - * Device-specific data; allocated in gasket_driver_desc.add_dev_cb() - * and freed in gasket_driver_desc.remove_dev_cb(). - */ - void *cb_data; - /* Protects access to per-device data (i.e. this structure). */ struct mutex mutex; @@ -415,29 +409,6 @@ struct gasket_driver_desc { int interrupt_pack_width; /* Driver callback functions - all may be NULL */ - /* - * add_dev_cb: Callback when a device is found. - * @dev: The gasket_dev struct for this driver instance. - * - * This callback should initialize the device-specific cb_data. - * Called when a device is found by the driver, - * before any BAR ranges have been mapped. If this call fails (returns - * nonzero), remove_dev_cb will be called. - * - */ - int (*add_dev_cb)(struct gasket_dev *dev); - - /* - * remove_dev_cb: Callback for when a device is removed from the system. - * @dev: The gasket_dev struct for this driver instance. - * - * This callback should free data allocated in add_dev_cb. - * Called immediately before a device is unregistered by the driver. - * All framework-managed resources will have been cleaned up by the time - * this callback is invoked (PCI BARs, character devices, ...). - */ - int (*remove_dev_cb)(struct gasket_dev *dev); - /* * device_open_cb: Callback for when a device node is opened in write * mode. From 18beeae8aabb4eddfb8a7212a0e76cfb0566f6a4 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:44 -0700 Subject: [PATCH 136/209] staging: gasket: apex: fold device add/remove logic inline Gasket device drivers are now in charge of the device add and remove sequences; the framework callbacks for these are deleted. Move the apex device add callback code to the probe function. Apex did not implement the removal callback. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 69 ++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index a4b051c..204c3eb 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -448,37 +448,6 @@ static int apex_reset(struct gasket_dev *gasket_dev) return ret; } -static int apex_add_dev_cb(struct gasket_dev *gasket_dev) -{ - ulong page_table_ready, msix_table_ready; - int retries = 0; - - apex_reset(gasket_dev); - - while (retries < APEX_RESET_RETRY) { - page_table_ready = - gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); - msix_table_ready = - gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); - if (page_table_ready && msix_table_ready) - break; - schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); - retries++; - } - - if (retries == APEX_RESET_RETRY) { - if (!page_table_ready) - dev_err(gasket_dev->dev, "Page table init timed out\n"); - if (!msix_table_ready) - dev_err(gasket_dev->dev, "MSI-X table init timed out\n"); - return -ETIMEDOUT; - } - - return 0; -} - /* * Check permissions for Apex ioctls. * Returns true if the current user may execute this ioctl, and false otherwise. @@ -690,6 +659,8 @@ static int apex_pci_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) { int ret; + ulong page_table_ready, msix_table_ready; + int retries = 0; struct gasket_dev *gasket_dev; ret = pci_enable_device(pci_dev); @@ -708,15 +679,42 @@ static int apex_pci_probe(struct pci_dev *pci_dev, } pci_set_drvdata(pci_dev, gasket_dev); + apex_reset(gasket_dev); + + while (retries < APEX_RESET_RETRY) { + page_table_ready = + gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); + msix_table_ready = + gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); + if (page_table_ready && msix_table_ready) + break; + schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); + retries++; + } + + if (retries == APEX_RESET_RETRY) { + if (!page_table_ready) + dev_err(gasket_dev->dev, "Page table init timed out\n"); + if (!msix_table_ready) + dev_err(gasket_dev->dev, "MSI-X table init timed out\n"); + ret = -ETIMEDOUT; + goto remove_device; + } + ret = gasket_enable_device(gasket_dev); if (ret) { dev_err(&pci_dev->dev, "error enabling gasket device\n"); - gasket_pci_remove_device(pci_dev); - pci_disable_device(pci_dev); - return ret; + goto remove_device; } return 0; + +remove_device: + gasket_pci_remove_device(pci_dev); + pci_disable_device(pci_dev); + return ret; } static void apex_pci_remove(struct pci_dev *pci_dev) @@ -761,9 +759,6 @@ static struct gasket_driver_desc apex_desc = { .interrupts = apex_interrupts, .interrupt_pack_width = 7, - .add_dev_cb = apex_add_dev_cb, - .remove_dev_cb = NULL, - .sysfs_setup_cb = apex_sysfs_setup_cb, .sysfs_cleanup_cb = NULL, From 1529ff2c6963dbc7c7b170ec5d9f3c71362ab90e Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:45 -0700 Subject: [PATCH 137/209] staging: gasket: core: remove sysfs setup and cleanup callbacks Gasket device drivers now call into the gasket framework to initialize and de-initialize, rather than the other way around. The calling code can perform sysfs setup and cleanup actions without callbacks from the framework. Remove the sysfs setup and cleanup callbacks. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 10 ---------- gasket_core.h | 18 ------------------ 2 files changed, 28 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 0d76e18..ace92f1 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1489,18 +1489,9 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, if (ret) goto fail4; - ret = check_and_invoke_callback(gasket_dev, - driver_desc->sysfs_setup_cb); - if (ret) { - dev_err(gasket_dev->dev, "Error in sysfs setup cb: %d\n", ret); - goto fail5; - } - *gasket_devp = gasket_dev; return 0; -fail5: - check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); fail4: fail3: gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); @@ -1550,7 +1541,6 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) gasket_cleanup_pci(gasket_dev); - check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb); gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); device_destroy(internal_desc->class, gasket_dev->dev_info.devt); gasket_free_dev(gasket_dev); diff --git a/gasket_core.h b/gasket_core.h index 0ef0a26..275fd0b 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -444,24 +444,6 @@ struct gasket_driver_desc { */ int (*device_close_cb)(struct gasket_dev *dev); - /* - * sysfs_setup_cb: Callback to set up driver-specific sysfs nodes. - * @dev: Pointer to the gasket_dev struct for this device. - * - * Called during the add gasket device call. - * - */ - int (*sysfs_setup_cb)(struct gasket_dev *dev); - - /* - * sysfs_cleanup_cb: Callback to clean up driver-specific sysfs nodes. - * @dev: Pointer to the gasket_dev struct for this device. - * - * Called during device disable processing. - * - */ - int (*sysfs_cleanup_cb)(struct gasket_dev *dev); - /* * get_mappable_regions_cb: Get descriptors of mappable device memory. * @gasket_dev: Pointer to the struct gasket_dev for this device. From 9117159d16a2f08a9343bbaa219f6e75d651efea Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:46 -0700 Subject: [PATCH 138/209] staging: gasket: apex: move sysfs setup code to probe function The gasket framework no longer provides callbacks to the device driver for sysfs setup and teardown. Move the sysfs setup code to the device probe function. Apex does not implement sysfs cleanup code. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 204c3eb..4c3129c 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -632,12 +632,6 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { GASKET_END_OF_ATTR_ARRAY }; -static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev) -{ - return gasket_sysfs_create_entries(gasket_dev->dev_info.device, - apex_sysfs_attrs); -} - /* On device open, perform a core reinit reset. */ static int apex_device_open_cb(struct gasket_dev *gasket_dev) { @@ -703,6 +697,11 @@ static int apex_pci_probe(struct pci_dev *pci_dev, goto remove_device; } + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, + apex_sysfs_attrs); + if (ret) + dev_err(&pci_dev->dev, "error creating device sysfs entries\n"); + ret = gasket_enable_device(gasket_dev); if (ret) { dev_err(&pci_dev->dev, "error enabling gasket device\n"); @@ -759,9 +758,6 @@ static struct gasket_driver_desc apex_desc = { .interrupts = apex_interrupts, .interrupt_pack_width = 7, - .sysfs_setup_cb = apex_sysfs_setup_cb, - .sysfs_cleanup_cb = NULL, - .device_open_cb = apex_device_open_cb, .device_close_cb = apex_device_cleanup, From 52f647e62e8ef4e815a7677dbebbaa3b6b59afcb Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:47 -0700 Subject: [PATCH 139/209] staging: gasket: core: protect against races during unregister Keep mutex held across the unregistration operation, until the driver_desc field of the global table is removed, to prevent a concurrent accessor from looking up the driver_desc while gasket_unregister_device() is in the processing of removing it. Reported-by: Guenter Roeck Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index ace92f1..a6462b6 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1765,9 +1765,9 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) break; } } - mutex_unlock(&g_mutex); if (!internal_desc) { + mutex_unlock(&g_mutex); pr_err("request to unregister unknown desc: %s, %d:%d\n", driver_desc->name, driver_desc->major, driver_desc->minor); @@ -1780,7 +1780,6 @@ void gasket_unregister_device(const struct gasket_driver_desc *driver_desc) class_destroy(internal_desc->class); /* Finally, effectively "remove" the driver. */ - mutex_lock(&g_mutex); g_descs[desc_idx].driver_desc = NULL; mutex_unlock(&g_mutex); From b9ded2cf72ae4efdcf4c8a653099dccff9c35522 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:48 -0700 Subject: [PATCH 140/209] staging: gasket: apex: place in low power reset until opened The apex device is left out of reset mode at the end of device probe/initialize processing. Add a call to enter reset at the end of the sequence, triggering power gating and other low power features. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apex_driver.c b/apex_driver.c index 4c3129c..9aa8408 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -708,6 +708,10 @@ static int apex_pci_probe(struct pci_dev *pci_dev, goto remove_device; } + /* Place device in low power mode until opened */ + if (allow_power_save) + apex_enter_reset(gasket_dev); + return 0; remove_device: From c3a0d069ba842ae66e4223f242e18f590e32de0a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Sun, 5 Aug 2018 13:07:49 -0700 Subject: [PATCH 141/209] staging: gasket: core: remove incorrect extraneous comment A copy-and-pasted comment from another code sequence is removed from gasket core init sequence. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index a6462b6..d12ab56 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1792,7 +1792,6 @@ static int __init gasket_init(void) int i; pr_debug("%s\n", __func__); - /* Check for duplicates and find a free slot. */ mutex_lock(&g_mutex); for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { g_descs[i].driver_desc = NULL; From 71f099b4c19e7a5cc3241f03d95ef325eaa98431 Mon Sep 17 00:00:00 2001 From: Sumit Pundir Date: Tue, 7 Aug 2018 16:14:28 +0530 Subject: [PATCH 142/209] staging: gasket: fix code indent for conditional statement Fixed a coding style issue related to indentation. Reported by checkpatch.pl Signed-off-by: Sumit Pundir Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index ed6ab3c..d4c5f8a 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1311,7 +1311,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, gasket_dev->coherent_buffer.virt_base = mem; *dma_address = driver_desc->coherent_buffer_description.base; - for (j = 0; j < num_pages; j++) { + for (j = 0; j < num_pages; j++) { gasket_dev->page_table[index]->coherent_pages[j].paddr = handle + j * PAGE_SIZE; gasket_dev->page_table[index]->coherent_pages[j].kernel_virt = From 084dc17032fce27a4232dd2fedc404dc8f647440 Mon Sep 17 00:00:00 2001 From: zhong jiang Date: Mon, 6 Aug 2018 11:10:19 +0800 Subject: [PATCH 143/209] staging: gasket: remove some extra semicolon That semicolons are unneeded, Just remove them. Signed-off-by: zhong jiang Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 09c3d07..1cfbc12 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -386,7 +386,7 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, "Cannot handle unsupported interrupt type %d\n", interrupt_data->type); ret = -EINVAL; - }; + } if (ret) { /* Failing to setup interrupts will cause the device to report @@ -445,7 +445,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) "Cannot handle unsupported interrupt type %d\n", gasket_dev->interrupt_data->type); ret = -EINVAL; - }; + } if (ret) { /* Failing to setup MSIx will cause the device @@ -493,7 +493,7 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) dev_dbg(gasket_dev->dev, "Cannot handle unsupported interrupt type %d\n", interrupt_data->type); - }; + } kfree(interrupt_data->interrupt_counts); kfree(interrupt_data->eventfd_ctxs); From 1fa1af899a3a75f372f1eae2afc3d51f24d92345 Mon Sep 17 00:00:00 2001 From: Sumit Kumar Date: Sun, 12 Aug 2018 12:38:19 +0530 Subject: [PATCH 144/209] staging: gasket: remove null ptr check before kfree Remove null ptr check before kfree because kfree is null ptr safe. Issue found by checkpatch. Signed-off-by: Sumit Kumar Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index d4c5f8a..bd921dc 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1328,10 +1328,8 @@ nomem: num_pages * PAGE_SIZE, mem, handle); } - if (gasket_dev->page_table[index]->coherent_pages) { - kfree(gasket_dev->page_table[index]->coherent_pages); - gasket_dev->page_table[index]->coherent_pages = NULL; - } + kfree(gasket_dev->page_table[index]->coherent_pages); + gasket_dev->page_table[index]->coherent_pages = NULL; gasket_dev->page_table[index]->num_coherent_pages = 0; return -ENOMEM; } From 6cf552635545b90dc1e3439aaebda67a5008c798 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 9 Aug 2018 11:44:32 +0100 Subject: [PATCH 145/209] staging: gasket: apex: remove unused array cm_mappable_regions Array cm_mappable_regions is defined but is never used hence it is redundant and can be removed. Cleans up clang warning: warning: 'cm_mappable_regions' defined but not used [-Wunused-const-variable=] Signed-off-by: Colin Ian King Signed-off-by: Greg Kroah-Hartman --- apex_driver.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 9aa8408..617841f 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -138,9 +138,6 @@ static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = { { 0x48000, 0x1000 }, }; -static const struct gasket_mappable_region cm_mappable_regions[1] = { { 0x0, - APEX_CH_MEM_BYTES } }; - /* Gasket device interrupts enums must be dense (i.e., no empty slots). */ enum apex_interrupt { APEX_INTERRUPT_INSTR_QUEUE = 0, From ceca0c9cb372da639e4ff2fe70670360911f9c0d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:20:57 -0700 Subject: [PATCH 146/209] staging: gasket: core: remove debug log that could crash A debug log in gasket_alloc_dev() is issued regardless of whether the device pointer used returned success or error. The log isn't that useful anyway, remove it. Signed-off-by: Todd Poynor Reviewed-by: Rob Springer Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index d12ab56..37d14e3 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -234,8 +234,6 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, dev_info->device = device_create(internal_desc->class, parent, dev_info->devt, gasket_dev, dev_info->name); - dev_dbg(dev_info->device, "Gasket device allocated.\n"); - /* cdev has not yet been added; cdev_added is 0 */ dev_info->gasket_dev_ptr = gasket_dev; /* ownership is all 0, indicating no owner or opens. */ From 68a1cca7c6b4fe89d0ce1b5a35b466aed89768e3 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:20:58 -0700 Subject: [PATCH 147/209] staging: gasket: core: fix line continuation indent in gasket_alloc_dev Previous cleanups missed a case of multi-line function call with line continuation parameters not aligned per kernel style. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 37d14e3..3fb8052 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -231,8 +231,9 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, dev_info->devt = MKDEV(driver_desc->major, driver_desc->minor + gasket_dev->dev_idx); - dev_info->device = device_create(internal_desc->class, parent, - dev_info->devt, gasket_dev, dev_info->name); + dev_info->device = + device_create(internal_desc->class, parent, dev_info->devt, + gasket_dev, dev_info->name); /* cdev has not yet been added; cdev_added is 0 */ dev_info->gasket_dev_ptr = gasket_dev; From c64522af66be1cf576952d4ab08002ab1971d0c7 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:20:59 -0700 Subject: [PATCH 148/209] staging: gasket: core: remove kobj_name param from gasket_alloc_dev gasket_alloc_dev can retrieve the device name from the parent parameter, a separate parameter isn't needed for this. Rename the variable to better reflect its meaning, as the name of the parent device for which a gasket device is being allocated. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 3fb8052..5f54b36 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -189,26 +189,26 @@ static int gasket_find_dev_slot(struct gasket_internal_desc *internal_desc, * Returns 0 if successful, a negative error code otherwise. */ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, - struct device *parent, struct gasket_dev **pdev, - const char *kobj_name) + struct device *parent, struct gasket_dev **pdev) { int dev_idx; const struct gasket_driver_desc *driver_desc = internal_desc->driver_desc; struct gasket_dev *gasket_dev; struct gasket_cdev_info *dev_info; + const char *parent_name = dev_name(parent); - pr_debug("Allocating a Gasket device %s.\n", kobj_name); + pr_debug("Allocating a Gasket device, parent %s.\n", parent_name); *pdev = NULL; - dev_idx = gasket_find_dev_slot(internal_desc, kobj_name); + dev_idx = gasket_find_dev_slot(internal_desc, parent_name); if (dev_idx < 0) return dev_idx; gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL); if (!gasket_dev) { - pr_err("no memory for device %s\n", kobj_name); + pr_err("no memory for device, parent %s\n", parent_name); return -ENOMEM; } internal_desc->devs[dev_idx] = gasket_dev; @@ -217,7 +217,7 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, gasket_dev->internal_desc = internal_desc; gasket_dev->dev_idx = dev_idx; - snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", kobj_name); + snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", parent_name); gasket_dev->dev = get_device(parent); /* gasket_bar_data is uninitialized. */ gasket_dev->num_page_tables = driver_desc->num_page_tables; @@ -1431,13 +1431,12 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, struct gasket_dev **gasket_devp) { int ret; - const char *kobj_name = dev_name(&pci_dev->dev); struct gasket_internal_desc *internal_desc; struct gasket_dev *gasket_dev; const struct gasket_driver_desc *driver_desc; struct device *parent; - pr_debug("add PCI device %s\n", kobj_name); + dev_dbg(&pci_dev->dev, "add PCI gasket device\n"); mutex_lock(&g_mutex); internal_desc = lookup_internal_desc(pci_dev); @@ -1451,7 +1450,7 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, driver_desc = internal_desc->driver_desc; parent = &pci_dev->dev; - ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); + ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev); if (ret) return ret; gasket_dev->pci_dev = pci_dev; From a6a849123520e8d3bdc259a796661ddb0988c560 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:00 -0700 Subject: [PATCH 149/209] staging: gasket: core: remove ftrace-style debug logs Remove debug logs that only indicate the name of the entered function, in favor of using ftrace for function tracing style logs. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 5f54b36..0fe5b86 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1789,7 +1789,6 @@ static int __init gasket_init(void) { int i; - pr_debug("%s\n", __func__); mutex_lock(&g_mutex); for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { g_descs[i].driver_desc = NULL; @@ -1804,7 +1803,6 @@ static int __init gasket_init(void) static void __exit gasket_exit(void) { - pr_debug("%s\n", __func__); } MODULE_DESCRIPTION("Google Gasket driver framework"); MODULE_VERSION(GASKET_FRAMEWORK_VERSION); From 89ff180f96dfbb9e5162f45d8a5eab5177944dc1 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:01 -0700 Subject: [PATCH 150/209] staging: gasket: remove gasket_exit() Remove now-empty gasket_exit() function. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 0fe5b86..aee819f 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1801,12 +1801,8 @@ static int __init gasket_init(void) return 0; } -static void __exit gasket_exit(void) -{ -} MODULE_DESCRIPTION("Google Gasket driver framework"); MODULE_VERSION(GASKET_FRAMEWORK_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Rob Springer "); module_init(gasket_init); -module_exit(gasket_exit); From 72caabea8ce5fd3987fc9552b8a876ae83107e48 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:03 -0700 Subject: [PATCH 151/209] staging: gasket: page table: use dma_mapping_error for error detection gasket_perform_mapping() call dma_mapping_error() to determine if mapping failed. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index bd921dc..4d24992 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -493,7 +493,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, (void *)page_to_pfn(page), (unsigned long long)ptes[i].dma_addr); - if (ptes[i].dma_addr == -1) { + if (dma_mapping_error(pg_tbl->device, + ptes[i].dma_addr)) { dev_dbg(pg_tbl->device, "%s i %d -> fail to map page %llx " "[pfn %p ohys %p]\n", From 4d13cb937ff8ee8cd62d90593aa3562cc008049a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:04 -0700 Subject: [PATCH 152/209] staging: gasket: core: switch to relaxed memory-mapped I/O Use of readl() is deprecated; readl_relaxed() with appropriate memory barriers is preferred. Switch to relaxed reads and writes for better performance as well. Memory barriers required for I/O vs. normal memory access on Apex devices have already been explicitly coded in the page table routines. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gasket_core.h b/gasket_core.h index 275fd0b..fd7e75b 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -590,25 +590,25 @@ const char *gasket_num_name_lookup(uint num, static inline ulong gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar, ulong location) { - return readq(&gasket_dev->bar_data[bar].virt_base[location]); + return readq_relaxed(&gasket_dev->bar_data[bar].virt_base[location]); } static inline void gasket_dev_write_64(struct gasket_dev *dev, u64 value, int bar, ulong location) { - writeq(value, &dev->bar_data[bar].virt_base[location]); + writeq_relaxed(value, &dev->bar_data[bar].virt_base[location]); } static inline void gasket_dev_write_32(struct gasket_dev *dev, u32 value, int bar, ulong location) { - writel(value, &dev->bar_data[bar].virt_base[location]); + writel_relaxed(value, &dev->bar_data[bar].virt_base[location]); } static inline u32 gasket_dev_read_32(struct gasket_dev *dev, int bar, ulong location) { - return readl(&dev->bar_data[bar].virt_base[location]); + return readl_relaxed(&dev->bar_data[bar].virt_base[location]); } static inline void gasket_read_modify_write_64(struct gasket_dev *dev, int bar, From e14033cdb6b7ec8cea40dc0ad6c9f89460c19c7a Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:05 -0700 Subject: [PATCH 153/209] staging: gasket: page table: remove extraneous memory barriers Some explicit memory barriers in the page table code are not necessary, either because: (a) The barrier follows a non-relaxed MMIO access that already performs a read or write memory barrier. (b) The barrier follows DMA API calls for which the device-visible effects of IOMMU programming are guaranteed to be flushed to the IOMMU prior to the call returning, and doesn't need to sync with normal memory access. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 4d24992..53492f4 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -317,8 +317,6 @@ static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl, /* Release the page table from the device */ writeq(0, slot); - /* Force sync around the address release. */ - mb(); if (pte->dma_addr) dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, @@ -504,8 +502,6 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, (void *)page_to_phys(page)); return -1; } - /* Wait until the page is mapped. */ - mb(); } /* Make the DMA-space address available to the device. */ @@ -604,12 +600,13 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, */ for (i = 0; i < num_pages; i++) { /* release the address from the device, */ - if (is_simple_mapping || ptes[i].status == PTE_INUSE) + if (is_simple_mapping || ptes[i].status == PTE_INUSE) { writeq(0, &slots[i]); - else + } else { ((u64 __force *)slots)[i] = 0; - /* Force sync around the address release. */ - mb(); + /* sync above PTE update before updating mappings */ + wmb(); + } /* release the address from the driver, */ if (ptes[i].status == PTE_INUSE) { @@ -898,8 +895,6 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, /* Map the page into DMA space. */ pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); - /* Wait until the page is mapped. */ - mb(); /* make the addresses available to the device */ dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; From 16c146bc73e6e029405aac4bbc89166de3617318 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:06 -0700 Subject: [PATCH 154/209] staging: gasket: core: factor out generic device add code from PCI code Split out generic gasket device add code from the code for adding a PCI gasket device, in prep for other gasket device types in the future. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 76 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index aee819f..ce8ae22 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1419,6 +1419,48 @@ int gasket_enable_device(struct gasket_dev *gasket_dev) } EXPORT_SYMBOL(gasket_enable_device); +static int __gasket_add_device(struct device *parent_dev, + struct gasket_internal_desc *internal_desc, + struct gasket_dev **gasket_devp) +{ + int ret; + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + + ret = gasket_alloc_dev(internal_desc, parent_dev, &gasket_dev); + if (ret) + return ret; + if (IS_ERR(gasket_dev->dev_info.device)) { + dev_err(parent_dev, "Cannot create %s device %s [ret = %ld]\n", + driver_desc->name, gasket_dev->dev_info.name, + PTR_ERR(gasket_dev->dev_info.device)); + ret = -ENODEV; + goto free_gasket_dev; + } + + ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device, + gasket_dev); + if (ret) + goto remove_device; + + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, + gasket_sysfs_generic_attrs); + if (ret) + goto remove_sysfs_mapping; + + *gasket_devp = gasket_dev; + return 0; + +remove_sysfs_mapping: + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); +remove_device: + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); +free_gasket_dev: + gasket_free_dev(gasket_dev); + return ret; +} + /* * Add PCI gasket device. * @@ -1433,7 +1475,6 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, int ret; struct gasket_internal_desc *internal_desc; struct gasket_dev *gasket_dev; - const struct gasket_driver_desc *driver_desc; struct device *parent; dev_dbg(&pci_dev->dev, "add PCI gasket device\n"); @@ -1447,29 +1488,15 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, return -ENODEV; } - driver_desc = internal_desc->driver_desc; - parent = &pci_dev->dev; - ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev); + ret = __gasket_add_device(parent, internal_desc, &gasket_dev); if (ret) return ret; - gasket_dev->pci_dev = pci_dev; - if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { - pr_err("Cannot create %s device %s [ret = %ld]\n", - driver_desc->name, gasket_dev->dev_info.name, - PTR_ERR(gasket_dev->dev_info.device)); - ret = -ENODEV; - goto fail1; - } + gasket_dev->pci_dev = pci_dev; ret = gasket_setup_pci(pci_dev, gasket_dev); if (ret) - goto fail2; - - ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device, - gasket_dev); - if (ret) - goto fail3; + goto cleanup_pci; /* * Once we've created the mapping structures successfully, attempt to @@ -1480,23 +1507,16 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, if (ret) { dev_err(gasket_dev->dev, "Cannot create sysfs pci link: %d\n", ret); - goto fail3; + goto cleanup_pci; } - ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, - gasket_sysfs_generic_attrs); - if (ret) - goto fail4; *gasket_devp = gasket_dev; return 0; -fail4: -fail3: - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); -fail2: +cleanup_pci: gasket_cleanup_pci(gasket_dev); + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); device_destroy(internal_desc->class, gasket_dev->dev_info.devt); -fail1: gasket_free_dev(gasket_dev); return ret; } From a74e602f32a077e863cb04907c61b3944a036a04 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:07 -0700 Subject: [PATCH 155/209] staging: gasket: core: factor out generic device remove code from PCI Separate code for generic parts of gasket device removal sequence from the PCI device removal code, in prep for non-PCI devices later. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index ce8ae22..5e048f6 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1461,6 +1461,14 @@ free_gasket_dev: return ret; } +static void __gasket_remove_device(struct gasket_internal_desc *internal_desc, + struct gasket_dev *gasket_dev) +{ + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); + gasket_free_dev(gasket_dev); +} + /* * Add PCI gasket device. * @@ -1515,9 +1523,7 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, cleanup_pci: gasket_cleanup_pci(gasket_dev); - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); - gasket_free_dev(gasket_dev); + __gasket_remove_device(internal_desc, gasket_dev); return ret; } EXPORT_SYMBOL(gasket_pci_add_device); @@ -1528,7 +1534,6 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) int i; struct gasket_internal_desc *internal_desc; struct gasket_dev *gasket_dev = NULL; - const struct gasket_driver_desc *driver_desc; /* Find the device desc. */ mutex_lock(&g_mutex); internal_desc = lookup_internal_desc(pci_dev); @@ -1538,8 +1543,6 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) } mutex_unlock(&g_mutex); - driver_desc = internal_desc->driver_desc; - /* Now find the specific device */ mutex_lock(&internal_desc->mutex); for (i = 0; i < GASKET_DEV_MAX; i++) { @@ -1558,10 +1561,7 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) internal_desc->driver_desc->name); gasket_cleanup_pci(gasket_dev); - - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); - gasket_free_dev(gasket_dev); + __gasket_remove_device(internal_desc, gasket_dev); } EXPORT_SYMBOL(gasket_pci_remove_device); From 29182ff7cb556ece07ba2bed891fc5b37fba732b Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:08 -0700 Subject: [PATCH 156/209] staging: gasket: core: rename lookup_internal_desc to be PCI-specific Rename lookup_internal_desc() to lookup_pci_internal_desc() to reflect use for PCI devices only, in prep for non-PCI devices in the future. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 5e048f6..99f3f11 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -651,13 +651,13 @@ void gasket_disable_device(struct gasket_dev *gasket_dev) EXPORT_SYMBOL(gasket_disable_device); /* - * Registered descriptor lookup. + * Registered driver descriptor lookup for PCI devices. * * Precondition: Called with g_mutex held (to avoid a race on return). * Returns NULL if no matching device was found. */ static struct gasket_internal_desc * -lookup_internal_desc(struct pci_dev *pci_dev) +lookup_pci_internal_desc(struct pci_dev *pci_dev) { int i; @@ -1488,7 +1488,7 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, dev_dbg(&pci_dev->dev, "add PCI gasket device\n"); mutex_lock(&g_mutex); - internal_desc = lookup_internal_desc(pci_dev); + internal_desc = lookup_pci_internal_desc(pci_dev); mutex_unlock(&g_mutex); if (!internal_desc) { dev_err(&pci_dev->dev, @@ -1536,7 +1536,7 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) struct gasket_dev *gasket_dev = NULL; /* Find the device desc. */ mutex_lock(&g_mutex); - internal_desc = lookup_internal_desc(pci_dev); + internal_desc = lookup_pci_internal_desc(pci_dev); if (!internal_desc) { mutex_unlock(&g_mutex); return; From ffa037c04728c7b4a3cd41771b6c334c20725178 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:09 -0700 Subject: [PATCH 157/209] staging: gasket: interrupt: refactor PCI MSIX-specific handler code Split interrupt handler into PCI MSIX-specific and generic functions, for adding non-MSIX handlers in the future. Move MSIX init code together,, out of generic init path. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_interrupt.c | 48 ++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 1cfbc12..f94e4ea 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -157,9 +157,22 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } } -static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) +static void +gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data, + int interrupt_index) { struct eventfd_ctx *ctx; + + trace_gasket_interrupt_event(interrupt_data->name, interrupt_index); + ctx = interrupt_data->eventfd_ctxs[interrupt_index]; + if (ctx) + eventfd_signal(ctx, 1); + + ++(interrupt_data->interrupt_counts[interrupt_index]); +} + +static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) +{ struct gasket_interrupt_data *interrupt_data = dev_id; int interrupt = -1; int i; @@ -175,14 +188,7 @@ static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) pr_err("Received unknown irq %d\n", irq); return IRQ_HANDLED; } - trace_gasket_interrupt_event(interrupt_data->name, interrupt); - - ctx = interrupt_data->eventfd_ctxs[interrupt]; - if (ctx) - eventfd_signal(ctx, 1); - - ++(interrupt_data->interrupt_counts[interrupt]); - + gasket_handle_interrupt(interrupt_data, interrupt); return IRQ_HANDLED; } @@ -192,6 +198,12 @@ gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data) int ret = 1; int i; + interrupt_data->msix_entries = + kcalloc(interrupt_data->num_interrupts, + sizeof(struct msix_entry), GFP_KERNEL); + if (!interrupt_data->msix_entries) + return -ENOMEM; + for (i = 0; i < interrupt_data->num_interrupts; i++) { interrupt_data->msix_entries[i].entry = i; interrupt_data->msix_entries[i].vector = 0; @@ -343,20 +355,10 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, interrupt_data->num_configured = 0; interrupt_data->wire_interrupt_offsets = wire_int_offsets; - /* Allocate all dynamic structures. */ - interrupt_data->msix_entries = kcalloc(num_interrupts, - sizeof(struct msix_entry), - GFP_KERNEL); - if (!interrupt_data->msix_entries) { - kfree(interrupt_data); - return -ENOMEM; - } - interrupt_data->eventfd_ctxs = kcalloc(num_interrupts, sizeof(struct eventfd_ctx *), GFP_KERNEL); if (!interrupt_data->eventfd_ctxs) { - kfree(interrupt_data->msix_entries); kfree(interrupt_data); return -ENOMEM; } @@ -366,7 +368,6 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, GFP_KERNEL); if (!interrupt_data->interrupt_counts) { kfree(interrupt_data->eventfd_ctxs); - kfree(interrupt_data->msix_entries); kfree(interrupt_data); return -ENOMEM; } @@ -417,6 +418,7 @@ gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) if (interrupt_data->msix_configured) pci_disable_msix(interrupt_data->pci_dev); interrupt_data->msix_configured = 0; + kfree(interrupt_data->msix_entries); } int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) @@ -448,10 +450,11 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) } if (ret) { - /* Failing to setup MSIx will cause the device + /* Failing to setup interrupts will cause the device * to report GASKET_STATUS_LAMED, but is not fatal. */ - dev_warn(gasket_dev->dev, "Couldn't init msix: %d\n", ret); + dev_warn(gasket_dev->dev, "Couldn't reinit interrupts: %d\n", + ret); return 0; } @@ -497,7 +500,6 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) kfree(interrupt_data->interrupt_counts); kfree(interrupt_data->eventfd_ctxs); - kfree(interrupt_data->msix_entries); kfree(interrupt_data); gasket_dev->interrupt_data = NULL; } From a6aafca18beff5653edeca611519f89043b38fcc Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:10 -0700 Subject: [PATCH 158/209] staging: gasket: interrupt: simplify interrupt init parameters Pass the gasket driver descriptor to the interrupt init function, rather than exploding out separate parameters from various fields of that structure. This allows us to make more localized changes to the types of interrupts supported (MSIX vs. wire, etc.) without affecting the calling sequence, and seems nicer for simplification purposes. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.c | 8 +------- gasket_interrupt.c | 27 +++++++++++++-------------- gasket_interrupt.h | 24 +----------------------- 3 files changed, 15 insertions(+), 44 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 99f3f11..f230bec 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1357,13 +1357,7 @@ int gasket_enable_device(struct gasket_dev *gasket_dev) const struct gasket_driver_desc *driver_desc = gasket_dev->internal_desc->driver_desc; - ret = gasket_interrupt_init(gasket_dev, driver_desc->name, - driver_desc->interrupt_type, - driver_desc->interrupts, - driver_desc->num_interrupts, - driver_desc->interrupt_pack_width, - driver_desc->interrupt_bar_index, - driver_desc->wire_interrupt_offsets); + ret = gasket_interrupt_init(gasket_dev); if (ret) { dev_err(gasket_dev->dev, "Critical failure to allocate interrupts: %d\n", ret); diff --git a/gasket_interrupt.c b/gasket_interrupt.c index f94e4ea..eb5dfbe 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -331,31 +331,30 @@ static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = { GASKET_END_OF_ATTR_ARRAY, }; -int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, - int type, - const struct gasket_interrupt_desc *interrupts, - int num_interrupts, int pack_width, int bar_index, - const struct gasket_wire_interrupt_offsets *wire_int_offsets) +int gasket_interrupt_init(struct gasket_dev *gasket_dev) { int ret; struct gasket_interrupt_data *interrupt_data; + const struct gasket_driver_desc *driver_desc = + gasket_get_driver_desc(gasket_dev); interrupt_data = kzalloc(sizeof(struct gasket_interrupt_data), GFP_KERNEL); if (!interrupt_data) return -ENOMEM; gasket_dev->interrupt_data = interrupt_data; - interrupt_data->name = name; - interrupt_data->type = type; + interrupt_data->name = driver_desc->name; + interrupt_data->type = driver_desc->interrupt_type; interrupt_data->pci_dev = gasket_dev->pci_dev; - interrupt_data->num_interrupts = num_interrupts; - interrupt_data->interrupts = interrupts; - interrupt_data->interrupt_bar_index = bar_index; - interrupt_data->pack_width = pack_width; + interrupt_data->num_interrupts = driver_desc->num_interrupts; + interrupt_data->interrupts = driver_desc->interrupts; + interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index; + interrupt_data->pack_width = driver_desc->interrupt_pack_width; interrupt_data->num_configured = 0; - interrupt_data->wire_interrupt_offsets = wire_int_offsets; + interrupt_data->wire_interrupt_offsets = + driver_desc->wire_interrupt_offsets; - interrupt_data->eventfd_ctxs = kcalloc(num_interrupts, + interrupt_data->eventfd_ctxs = kcalloc(driver_desc->num_interrupts, sizeof(struct eventfd_ctx *), GFP_KERNEL); if (!interrupt_data->eventfd_ctxs) { @@ -363,7 +362,7 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, return -ENOMEM; } - interrupt_data->interrupt_counts = kcalloc(num_interrupts, + interrupt_data->interrupt_counts = kcalloc(driver_desc->num_interrupts, sizeof(ulong), GFP_KERNEL); if (!interrupt_data->interrupt_counts) { diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 835af43..85526a1 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -24,30 +24,8 @@ struct gasket_interrupt_data; /* * Initialize the interrupt module. * @gasket_dev: The Gasket device structure for the device to be initted. - * @type: Type of the interrupt. (See gasket_interrupt_type). - * @name: The name to associate with these interrupts. - * @interrupts: An array of all interrupt descriptions for this device. - * @num_interrupts: The length of the @interrupts array. - * @pack_width: The width, in bits, of a single field in a packed interrupt reg. - * @bar_index: The bar containing all interrupt registers. - * - * Allocates and initializes data to track interrupt state for a device. - * After this call, no interrupts will be configured/delivered; call - * gasket_interrupt_set_vector[_packed] to associate each interrupt with an - * __iomem location, then gasket_interrupt_set_eventfd to associate an eventfd - * with an interrupt. - * - * If num_interrupts interrupts are not available, this call will return a - * negative error code. In that case, gasket_interrupt_cleanup should still be - * called. Returns 0 on success (which can include a device where interrupts - * are not possible to set up, but is otherwise OK; that device will report - * status LAMED.) */ -int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, - int type, - const struct gasket_interrupt_desc *interrupts, - int num_interrupts, int pack_width, int bar_index, - const struct gasket_wire_interrupt_offsets *wire_int_offsets); +int gasket_interrupt_init(struct gasket_dev *gasket_dev); /* * Clean up a device's interrupt structure. From c1f31b7544dd43b54e6376b78ea2846a1e5ea3aa Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 Aug 2018 20:21:11 -0700 Subject: [PATCH 159/209] staging: gasket: interrupt: remove unimplemented interrupt types Interrupt types PCI_MSI and PLATFORM_WIRE are unused and unimplemented. Remove these. Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_core.h | 11 ----------- gasket_interrupt.c | 34 +--------------------------------- 2 files changed, 1 insertion(+), 44 deletions(-) diff --git a/gasket_core.h b/gasket_core.h index fd7e75b..0203460 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -50,8 +50,6 @@ enum gasket_interrupt_packing { /* Type of the interrupt supported by the device. */ enum gasket_interrupt_type { PCI_MSIX = 0, - PCI_MSI = 1, - PLATFORM_WIRE = 2, }; /* @@ -69,12 +67,6 @@ struct gasket_interrupt_desc { int packing; }; -/* Offsets to the wire interrupt handling registers */ -struct gasket_wire_interrupt_offsets { - u64 pending_bit_array; - u64 mask_array; -}; - /* * This enum is used to identify memory regions being part of the physical * memory that belongs to a device. @@ -384,9 +376,6 @@ struct gasket_driver_desc { */ struct gasket_coherent_buffer_desc coherent_buffer_description; - /* Offset of wire interrupt registers. */ - const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets; - /* Interrupt type. (One of gasket_interrupt_type). */ int interrupt_type; diff --git a/gasket_interrupt.c b/gasket_interrupt.c index eb5dfbe..2cd262b 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -45,9 +45,6 @@ struct gasket_interrupt_data { /* The width of a single interrupt in a packed interrupt register. */ int pack_width; - /* offset of wire interrupt registers */ - const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets; - /* * Design-wise, these elements should be bundled together, but * pci_enable_msix's interface requires that they be managed @@ -92,19 +89,6 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) dev_dbg(gasket_dev->dev, "Running interrupt setup\n"); - if (interrupt_data->type == PLATFORM_WIRE || - interrupt_data->type == PCI_MSI) { - /* Nothing needs to be done for platform or PCI devices. */ - return; - } - - if (interrupt_data->type != PCI_MSIX) { - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); - return; - } - /* Setup the MSIX table. */ for (i = 0; i < interrupt_data->num_interrupts; i++) { @@ -351,8 +335,6 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev) interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index; interrupt_data->pack_width = driver_desc->interrupt_pack_width; interrupt_data->num_configured = 0; - interrupt_data->wire_interrupt_offsets = - driver_desc->wire_interrupt_offsets; interrupt_data->eventfd_ctxs = kcalloc(driver_desc->num_interrupts, sizeof(struct eventfd_ctx *), @@ -379,12 +361,7 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev) force_msix_interrupt_unmasking(gasket_dev); break; - case PCI_MSI: - case PLATFORM_WIRE: default: - dev_err(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); ret = -EINVAL; } @@ -439,12 +416,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) force_msix_interrupt_unmasking(gasket_dev); break; - case PCI_MSI: - case PLATFORM_WIRE: default: - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - gasket_dev->interrupt_data->type); ret = -EINVAL; } @@ -489,12 +461,8 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) gasket_interrupt_msix_cleanup(interrupt_data); break; - case PCI_MSI: - case PLATFORM_WIRE: default: - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); + break; } kfree(interrupt_data->interrupt_counts); From dd3fb175c5f4daa41c8f21cc5d1376c3f8fb0676 Mon Sep 17 00:00:00 2001 From: Alex Van Damme Date: Tue, 4 Sep 2018 12:31:36 -0700 Subject: [PATCH 160/209] Cleanup apex performance patch Change-Id: I9a419ef63cee937b7b4c183c9070be011555f8d4 --- apex_driver.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 617841f..e34b34e 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -27,7 +27,7 @@ /* Constants */ #define APEX_DEVICE_NAME "Apex" -#define APEX_DRIVER_VERSION "1.0" +#define APEX_DRIVER_VERSION "1.1" /* CSRs are in BAR 2. */ #define APEX_BAR_INDEX 2 @@ -489,7 +489,9 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, } /* apex_set_performance_expectation: Adjust clock rates for Apex. */ -static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulong arg) +static long apex_set_performance_expectation( + struct gasket_dev *gasket_dev, + struct apex_performance_expectation_ioctl __user *argp) { struct apex_performance_expectation_ioctl ibuf; uint32_t rg_gcb_clk_div = 0; @@ -503,7 +505,7 @@ static long apex_set_performance_expectation(struct gasket_dev *gasket_dev, ulon if (bypass_top_level) return 0; - if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf))) + if (copy_from_user(&ibuf, argp, sizeof(ibuf))) return -EFAULT; switch (ibuf.performance) { From 75b8025f7e35766c0a546ccc6c43556181e41525 Mon Sep 17 00:00:00 2001 From: Jae Yoo Date: Wed, 17 Oct 2018 14:31:31 +0900 Subject: [PATCH 161/209] Add arm32 compatibility into gasket driver Add a compat_ioctl handler for arm32 and replace ulong to u64. Bug: 115850337, 117133232 Test: label_image runs correctly. Change-Id: Ie17244faff2aa1461397e740fba4581c6b112f30 --- gasket_core.c | 1 + gasket_core.h | 8 +++++++- gasket_ioctl.c | 10 ++++++++-- gasket_page_table.c | 44 ++++++++++++++++++++++---------------------- gasket_page_table.h | 6 +++--- 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index f230bec..fe55ab0 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1347,6 +1347,7 @@ static const struct file_operations gasket_file_ops = { .open = gasket_open, .release = gasket_release, .unlocked_ioctl = gasket_ioctl, + .compat_ioctl = gasket_ioctl, }; /* Perform final init and marks the device as active. */ diff --git a/gasket_core.h b/gasket_core.h index 0203460..56805d3 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -17,6 +17,12 @@ #include #include +#define writeq_relaxed writeq +#define writeq(__value, __reg) \ + (*(volatile u64 __force *)(__reg) = (cpu_to_le64(__value))) +#define readq_relaxed readq +#define readq(__reg) le64_to_cpu(*(volatile u64 __force *)(__reg)) + #include "gasket_constants.h" /** @@ -576,7 +582,7 @@ const char *gasket_num_name_lookup(uint num, const struct gasket_num_name *table); /* Handy inlines */ -static inline ulong gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar, +static inline u64 gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar, ulong location) { return readq_relaxed(&gasket_dev->bar_data[bar].virt_base[location]); diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 0ca48e6..d32bce9 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -191,6 +191,7 @@ static int gasket_config_coherent_allocator( { int ret; struct gasket_coherent_alloc_config_ioctl ibuf; + dma_addr_t dma_address; if (copy_from_user(&ibuf, argp, sizeof(struct gasket_coherent_alloc_config_ioctl))) @@ -206,16 +207,21 @@ static int gasket_config_coherent_allocator( return -ENOMEM; if (ibuf.enable == 0) { + dma_address = ibuf.dma_address; ret = gasket_free_coherent_memory(gasket_dev, ibuf.size, - ibuf.dma_address, + dma_address, ibuf.page_table_index); } else { ret = gasket_alloc_coherent_memory(gasket_dev, ibuf.size, - &ibuf.dma_address, + &dma_address, ibuf.page_table_index); } if (ret) return ret; + + if (ibuf.enable != 0) + ibuf.dma_address = dma_address; + if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; diff --git a/gasket_page_table.c b/gasket_page_table.c index 53492f4..3aff2aa 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -143,7 +143,7 @@ struct gasket_coherent_page_entry { u64 user_virt; /* User virtual address that was mapped by the mmap kernel subsystem */ - u64 kernel_virt; + dma_addr_t kernel_virt; /* * Whether this page has been mapped into a user land process virtual @@ -528,7 +528,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, * Does not perform validity checking. */ static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) + u64 dev_addr) { return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & (pg_tbl->config.total_entries - 1); @@ -539,7 +539,7 @@ static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl, * Does not perform validity checking. */ static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) + u64 dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); @@ -550,7 +550,7 @@ static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, * Does not perform validity checking. */ static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) + u64 dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & (GASKET_PAGES_PER_SUBTABLE - 1); @@ -561,7 +561,7 @@ static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { if (!gasket_is_pte_range_free(pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr), @@ -629,7 +629,7 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); @@ -642,7 +642,7 @@ static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { uint slot_idx, remain, len; struct gasket_page_table_entry *pte; @@ -673,7 +673,7 @@ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, /* Evaluates to nonzero if the specified virtual address is simple. */ static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, - ulong addr) + u64 addr) { return !((addr) & (pg_tbl)->extended_flag); } @@ -724,7 +724,7 @@ static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, * currently-partitioned simple pages. */ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { ulong page_offset = dev_addr & (PAGE_SIZE - 1); ulong page_index = @@ -732,7 +732,7 @@ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, if (gasket_components_to_dev_address(pg_tbl, 1, page_index, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", + dev_err(pg_tbl->device, "address is invalid, 0x%llX\n", dev_addr); return true; } @@ -762,18 +762,18 @@ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, * currently-partitioned extended pages. */ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { /* Starting byte index of dev_addr into the first mapped page */ ulong page_offset = dev_addr & (PAGE_SIZE - 1); ulong page_global_idx, page_lvl0_idx; ulong num_lvl0_pages; - ulong addr; + u64 addr; /* check if the device address is out of bound */ addr = dev_addr & ~((pg_tbl)->extended_flag); if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n", + dev_err(pg_tbl->device, "device address out of bounds: 0x%llx\n", dev_addr); return true; } @@ -791,7 +791,7 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", + dev_err(pg_tbl->device, "address is invalid: 0x%llx\n", dev_addr); return true; } @@ -819,7 +819,7 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { if (!num_pages) return; @@ -844,7 +844,7 @@ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); if (ret) { dev_err(pg_tbl->device, - "page table slots %u (@ 0x%lx) to %u are not available\n", + "page table slots %u (@ 0x%llx) to %u are not available\n", slot_idx, dev_addr, slot_idx + num_pages - 1); return ret; } @@ -920,7 +920,7 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_entries) + u64 dev_addr, uint num_entries) { int ret = 0; uint remain, subtable_slot_idx, len; @@ -978,7 +978,7 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, if (ret) { dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; dev_err(pg_tbl->device, - "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are " + "page table slots (%lu,%llu) (@ 0x%lx) to (%lu,%lu) are " "not available\n", gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), dev_addr, @@ -1061,7 +1061,7 @@ EXPORT_SYMBOL(gasket_page_table_map); * * The page table mutex is held for the entire operation. */ -void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr, +void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, u64 dev_addr, uint num_pages) { if (!num_pages) @@ -1106,7 +1106,7 @@ void gasket_page_table_reset(struct gasket_page_table *pg_tbl) /* See gasket_page_table.h for description. */ int gasket_page_table_lookup_page( - struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage, + struct gasket_page_table *pg_tbl, u64 dev_addr, struct page **ppage, ulong *poffset) { uint page_num; @@ -1168,7 +1168,7 @@ EXPORT_SYMBOL(gasket_page_table_are_addrs_bad); /* See gasket_page_table.h for description. */ bool gasket_page_table_is_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes) + struct gasket_page_table *pg_tbl, u64 dev_addr, ulong bytes) { uint num_pages = bytes / PAGE_SIZE; @@ -1311,7 +1311,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, gasket_dev->page_table[index]->coherent_pages[j].paddr = handle + j * PAGE_SIZE; gasket_dev->page_table[index]->coherent_pages[j].kernel_virt = - (u64)mem + j * PAGE_SIZE; + (dma_addr_t)mem + j * PAGE_SIZE; } if (*dma_address == 0) diff --git a/gasket_page_table.h b/gasket_page_table.h index 7b01b73..783f594 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -106,7 +106,7 @@ int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, * Description: The inverse of gasket_map_pages. Unmaps pages from the device. */ void gasket_page_table_unmap(struct gasket_page_table *page_table, - ulong dev_addr, uint num_pages); + u64 dev_addr, uint num_pages); /* * Unmap ALL host pages from device memory. @@ -146,7 +146,7 @@ void gasket_page_table_garbage_collect(struct gasket_page_table *page_table); * and offset are returned through the pointers, if successful. */ int gasket_page_table_lookup_page(struct gasket_page_table *page_table, - ulong dev_addr, struct page **page, + u64 dev_addr, struct page **page, ulong *poffset); /* @@ -179,7 +179,7 @@ bool gasket_page_table_are_addrs_bad(struct gasket_page_table *page_table, * Returns true if the address is bad, false otherwise. */ bool gasket_page_table_is_dev_addr_bad(struct gasket_page_table *page_table, - ulong dev_addr, ulong bytes); + u64 dev_addr, ulong bytes); /* * Gets maximum size for the given page table. From 641f4139f4a9d4cb78a2c63ee9d61e62ddd8ee59 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Tue, 15 Jan 2019 14:20:24 -0800 Subject: [PATCH 162/209] Address b/117119472 Change-Id: I88c018628768b56e5904d89db2bd1d767078c4d1 --- apex_driver.c | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index e34b34e..206c890 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -495,12 +495,12 @@ static long apex_set_performance_expectation( { struct apex_performance_expectation_ioctl ibuf; uint32_t rg_gcb_clk_div = 0; - uint32_t rg_axi_clk_125m = 0; - const int AXI_CLK_125M_SHIFT = 2; - const int MCU_CLK_250M_SHIFT = 3; + uint32_t rg_axi_clk_fixed = 0; + const int AXI_CLK_FIXED_SHIFT = 2; + const int MCU_CLK_FIXED_SHIFT = 3; - // 8051 clock is always 250 MHz for PCIe, as it's not used at all. - const uint32_t rg_8051_clk_250m = 1; + // 8051 clock is fixed for PCIe, as it's not used at all. + const uint32_t rg_8051_clk_fixed = 1; if (bypass_top_level) return 0; @@ -510,31 +510,23 @@ static long apex_set_performance_expectation( switch (ibuf.performance) { case APEX_PERFORMANCE_LOW: - // - GCB clock: 62.5 MHz - // - AXI clock: 125 MHz rg_gcb_clk_div = 3; - rg_axi_clk_125m = 0; + rg_axi_clk_fixed = 0; break; case APEX_PERFORMANCE_MED: - // - GCB clock: 125 MHz - // - AXI clock: 125 MHz rg_gcb_clk_div = 2; - rg_axi_clk_125m = 0; + rg_axi_clk_fixed = 0; break; case APEX_PERFORMANCE_HIGH: - // - GCB clock: 250 MHz - // - AXI clock: 125 MHz rg_gcb_clk_div = 1; - rg_axi_clk_125m = 0; + rg_axi_clk_fixed = 0; break; case APEX_PERFORMANCE_MAX: - // - GCB clock: 500 MHz - // - AXI clock: 125 MHz rg_gcb_clk_div = 0; - rg_axi_clk_125m = 0; + rg_axi_clk_fixed = 0; break; default: @@ -546,7 +538,7 @@ static long apex_set_performance_expectation( */ gasket_read_modify_write_32( gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, - (rg_gcb_clk_div | (rg_axi_clk_125m << AXI_CLK_125M_SHIFT) | (rg_8051_clk_250m << MCU_CLK_250M_SHIFT)), + (rg_gcb_clk_div | (rg_axi_clk_fixed << AXI_CLK_FIXED_SHIFT) | (rg_8051_clk_fixed << MCU_CLK_FIXED_SHIFT)), /*mask_width=*/4, /*mask_shift=*/28); return 0; From 068cb76fc59185b980f3ee8e207b00c46c097b2f Mon Sep 17 00:00:00 2001 From: Nick Ewalt Date: Tue, 25 Sep 2018 09:52:02 -0700 Subject: [PATCH 163/209] staging: gasket: apex: fix sysfs_show sysfs_show was incorrectly extracting the sysfs_attribute_type from the gasket_sysfs_attribute. This prevented dispatch from working properly. Change-Id: Ib962a10cf24bd4be416708e0a6294b6db431c76f Signed-off-by: Nick Ewalt --- apex_driver.c | 2 +- gasket_sysfs.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 206c890..8904551 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -584,7 +584,7 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, return -ENODEV; } - type = (enum sysfs_attribute_type)gasket_sysfs_get_attr(device, attr); + type = (enum sysfs_attribute_type)gasket_attr->data.attr_type; switch (type) { case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE: ret = scnprintf(buf, PAGE_SIZE, "%u\n", diff --git a/gasket_sysfs.h b/gasket_sysfs.h index f32eaf8..151e8ed 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -152,8 +152,8 @@ void gasket_sysfs_put_device_data(struct device *device, * Returns the Gasket sysfs attribute associated with the kernel device * attribute and device structure itself. Upon success, this call will take a * reference to internal sysfs data that must be released with a call to - * gasket_sysfs_get_device_data. While this reference is held, the underlying - * device sysfs information/structure will remain valid/will not be deleted. + * gasket_sysfs_put_attr. While this reference is held, the underlying device + * sysfs information/structure will remain valid/will not be deleted. */ struct gasket_sysfs_attribute * gasket_sysfs_get_attr(struct device *device, struct device_attribute *attr); From e02a5548264bdf5ae9175bf1dd97760e6aac37d7 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 11 Feb 2019 13:26:44 -0800 Subject: [PATCH 164/209] Add sysfs node with temperature sensor reading. Change-Id: Ic9a71f1b1aaee64b68224cdc20dcfe2980b9c7d8 --- apex_driver.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/apex_driver.c b/apex_driver.c index 8904551..e227017 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -67,6 +67,7 @@ enum sysfs_attribute_type { ATTR_KERNEL_HIB_PAGE_TABLE_SIZE, ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES, + ATTR_TEMP, }; /* @@ -98,6 +99,9 @@ enum apex_bar2_regs { APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0, APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000, APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000, + APEX_BAR2_REG_OMC0_D0 = 0x01a0d0, + APEX_BAR2_REG_OMC0_D8 = 0x01a0d8, + APEX_BAR2_REG_OMC0_DC = 0x01a0dc, /* Error registers - Used mostly for debug */ APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0, @@ -601,6 +605,15 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, gasket_page_table_num_active_pages( gasket_dev->page_table[0])); break; + case ATTR_TEMP: + // Read temperature + ret = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC); + ret = (ret >> 16) & ((1 << 10) - 1); + // Remap to millicelsius (rough calculation) + ret = (((662 - ret) * 1000 / 4 + 500) * 10 + 500) / 10; + ret = snprintf(buf, PAGE_SIZE, "%i\n", ret); + break; default: dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", attr->attr.name); @@ -620,6 +633,7 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE), GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), + GASKET_SYSFS_RO(temp, sysfs_show, ATTR_TEMP), GASKET_END_OF_ATTR_ARRAY }; @@ -688,6 +702,20 @@ static int apex_pci_probe(struct pci_dev *pci_dev, goto remove_device; } + // Enable thermal sensor clocks + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D0, 0x1, 1, 7); + + // Enable thermal sensor (ENAD ENVR ENBG) + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, 0x7, 3, 0); + + // Enable OMC thermal sensor controller + // This bit should be asserted 100 us after ENAD ENVR ENBG + schedule_timeout(usecs_to_jiffies(100)); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC, 0x1, 1, 0); + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, apex_sysfs_attrs); if (ret) From c4a2c4870adefca175a4ece214b070904c7c5c2e Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 25 Mar 2019 17:57:15 -0700 Subject: [PATCH 165/209] staging: gasket: Fix pci remove/rescan for kernel module build. For driver build as module pci_fixup_header approach doesn't work. This causes apex pci class to remain 0xff and BARs remain unclaimed. Adding a retry with explicit fixup function call workarounds the problem. Bug: 129287752 Change-Id: Id5ff85ef427d3740706a214d3beb1c9bbf42ab5a Signed-off-by: Leonid Lobachev --- apex_driver.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apex_driver.c b/apex_driver.c index e227017..1eff202 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -652,7 +652,7 @@ static void apex_pci_fixup_class(struct pci_dev *pdev) pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class; } DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, - PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); + PCI_ANY_ID, 8, apex_pci_fixup_class); static int apex_pci_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) @@ -663,6 +663,13 @@ static int apex_pci_probe(struct pci_dev *pci_dev, struct gasket_dev *gasket_dev; ret = pci_enable_device(pci_dev); +#ifdef MODULE + if (ret) { + apex_pci_fixup_class(pci_dev); + pci_bus_assign_resources(pci_dev->bus); + ret = pci_enable_device(pci_dev); + } +#endif if (ret) { dev_err(&pci_dev->dev, "error enabling PCI device\n"); return ret; From 56f0b552fe41b80bcb558f1542b03d5f5af4f366 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Fri, 26 Apr 2019 11:47:56 -0700 Subject: [PATCH 166/209] staging: gasket: Add thermal sysfs nodes. Add node to read current temperature and nodes to set and enable/disable thermal warnings. Change-Id: I8a12f6e33679573107686fe32be2e4a8bb0f9cc7 Signed-off-by: Leonid Lobachev --- apex_driver.c | 125 +++++++++++++++++++++++++++++++++++++++++++++---- gasket_sysfs.h | 7 +++ 2 files changed, 122 insertions(+), 10 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 1eff202..d1b823e 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -68,6 +68,10 @@ enum sysfs_attribute_type { ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES, ATTR_TEMP, + ATTR_TEMP_WARN1, + ATTR_TEMP_WARN1_EN, + ATTR_TEMP_WARN2, + ATTR_TEMP_WARN2_EN, }; /* @@ -100,6 +104,7 @@ enum apex_bar2_regs { APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000, APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000, APEX_BAR2_REG_OMC0_D0 = 0x01a0d0, + APEX_BAR2_REG_OMC0_D4 = 0x01a0d4, APEX_BAR2_REG_OMC0_D8 = 0x01a0d8, APEX_BAR2_REG_OMC0_DC = 0x01a0dc, @@ -566,11 +571,23 @@ static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) } } +/* Linear fit optimized for 25C-100C */ +static int adc_to_millic(int adc) +{ + return (662 - adc) * 250 + 550; +} + +static int millic_to_adc(int millic) +{ + return (550 - millic) / 250 + 662; +} + /* Display driver sysfs entries. */ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, char *buf) { int ret; + unsigned value; struct gasket_dev *gasket_dev; struct gasket_sysfs_attribute *gasket_attr; enum sysfs_attribute_type type; @@ -606,13 +623,95 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, gasket_dev->page_table[0])); break; case ATTR_TEMP: - // Read temperature - ret = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_DC); - ret = (ret >> 16) & ((1 << 10) - 1); - // Remap to millicelsius (rough calculation) - ret = (((662 - ret) * 1000 / 4 + 500) * 10 + 500) / 10; - ret = snprintf(buf, PAGE_SIZE, "%i\n", ret); + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC); + value = (value >> 16) & ((1 << 10) - 1); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + break; + case ATTR_TEMP_WARN1: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4); + value = (value >> 16) & ((1 << 10) - 1); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + break; + case ATTR_TEMP_WARN2: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8); + value = (value >> 16) & ((1 << 10) - 1); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + break; + case ATTR_TEMP_WARN1_EN: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); + break; + case ATTR_TEMP_WARN2_EN: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); + break; + default: + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + +/* Set driver sysfs entries. */ +static ssize_t sysfs_store(struct device *device, struct device_attribute *attr, + const char *buf, size_t count) +{ + int ret = count, value; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + enum sysfs_attribute_type type; + + if (kstrtoint(buf, 10, &value)) + return -EINVAL; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + dev_err(device, "No Apex device sysfs mapping found\n"); + return -ENODEV; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + dev_err(device, "No Apex device sysfs attr data found\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + + type = (enum sysfs_attribute_type)gasket_attr->data.attr_type; + switch (type) { + case ATTR_TEMP_WARN1: + value = millic_to_adc(value); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, value, 10, + 16); + break; + case ATTR_TEMP_WARN2: + value = millic_to_adc(value); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, value, 10, + 16); + break; + case ATTR_TEMP_WARN1_EN: + value = value > 0 ? 1 : 0; + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, value, 1, + 31); + break; + case ATTR_TEMP_WARN2_EN: + value = value > 0 ? 1 : 0; + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, value, 1, + 31); break; default: dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", @@ -634,6 +733,12 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), GASKET_SYSFS_RO(temp, sysfs_show, ATTR_TEMP), + GASKET_SYSFS_RW(temp_warn1, sysfs_show, sysfs_store, ATTR_TEMP_WARN1), + GASKET_SYSFS_RW(temp_warn1_en, sysfs_show, sysfs_store, + ATTR_TEMP_WARN1_EN), + GASKET_SYSFS_RW(temp_warn2, sysfs_show, sysfs_store, ATTR_TEMP_WARN2), + GASKET_SYSFS_RW(temp_warn2_en, sysfs_show, sysfs_store, + ATTR_TEMP_WARN2_EN), GASKET_END_OF_ATTR_ARRAY }; @@ -711,17 +816,17 @@ static int apex_pci_probe(struct pci_dev *pci_dev, // Enable thermal sensor clocks gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D0, 0x1, 1, 7); + APEX_BAR2_REG_OMC0_D0, 0x1, 1, 7); // Enable thermal sensor (ENAD ENVR ENBG) gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D8, 0x7, 3, 0); + APEX_BAR2_REG_OMC0_D8, 0x7, 3, 0); // Enable OMC thermal sensor controller // This bit should be asserted 100 us after ENAD ENVR ENBG schedule_timeout(usecs_to_jiffies(100)); gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_DC, 0x1, 1, 0); + APEX_BAR2_REG_OMC0_DC, 0x1, 1, 0); ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, apex_sysfs_attrs); diff --git a/gasket_sysfs.h b/gasket_sysfs.h index 151e8ed..e8f29a3 100644 --- a/gasket_sysfs.h +++ b/gasket_sysfs.h @@ -79,6 +79,13 @@ struct gasket_sysfs_attribute { .data.attr_type = _attr_type \ } +#define GASKET_SYSFS_RW(_name, _show_function, _store_function, _attr_type) \ + { \ + .attr = __ATTR(_name, S_IWUSR | S_IWGRP | S_IRUGO, \ + _show_function, _store_function), \ + .data.attr_type = _attr_type \ + } + /* Initializes the Gasket sysfs subsystem. * * Description: Performs one-time initialization. Must be called before usage From 83ce758fc41bcad25a1aa8995849a8578e31d2ad Mon Sep 17 00:00:00 2001 From: Rob Springer Date: Tue, 28 May 2019 11:27:21 -0700 Subject: [PATCH 167/209] staging: gasket: release taken eventfd contexts Port of change by Rob Springer to fix leak of eventfd context objects in gasket framework interrupt code. Bug: 131928433 Bug: 80495595 Reported-by: Jann Horn Reported-by: Neel Natu Test: Resnet50PB, PortraitSegmentation Change-Id: I6bdea5ab4d3a7435e7f51c96817f0542d185b22c Signed-off-by: Todd Poynor --- gasket_interrupt.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 2cd262b..3cf7235 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #ifdef GASKET_KERNEL_TRACE_SUPPORT #define CREATE_TRACE_POINTS @@ -60,6 +61,9 @@ struct gasket_interrupt_data { /* The eventfd "callback" data for each interrupt. */ struct eventfd_ctx **eventfd_ctxs; + /* Spinlock to protect read/write races to eventfd_ctxs. */ + rwlock_t eventfd_ctx_lock; + /* The number of times each interrupt has been called. */ ulong *interrupt_counts; @@ -148,9 +152,11 @@ gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data, struct eventfd_ctx *ctx; trace_gasket_interrupt_event(interrupt_data->name, interrupt_index); + read_lock(&interrupt_data->eventfd_ctx_lock); ctx = interrupt_data->eventfd_ctxs[interrupt_index]; if (ctx) eventfd_signal(ctx, 1); + read_unlock(&interrupt_data->eventfd_ctx_lock); ++(interrupt_data->interrupt_counts[interrupt_index]); } @@ -353,6 +359,8 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev) return -ENOMEM; } + rwlock_init(&interrupt_data->eventfd_ctx_lock); + switch (interrupt_data->type) { case PCI_MSIX: ret = gasket_interrupt_msix_init(interrupt_data); @@ -386,9 +394,11 @@ gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) { int i; - for (i = 0; i < interrupt_data->num_configured; i++) + for (i = 0; i < interrupt_data->num_configured; i++) { + gasket_interrupt_clear_eventfd(interrupt_data, i); free_irq(interrupt_data->msix_entries[i].vector, interrupt_data); + } interrupt_data->num_configured = 0; if (interrupt_data->msix_configured) @@ -496,24 +506,38 @@ int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data, int interrupt, int event_fd) { - struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd); - - if (IS_ERR(ctx)) - return PTR_ERR(ctx); + struct eventfd_ctx *ctx; + ulong flags; if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; + ctx = eventfd_ctx_fdget(event_fd); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + /* Put the old eventfd ctx before setting, else we leak the ref. */ + write_lock_irqsave(&interrupt_data->eventfd_ctx_lock, flags); + if (interrupt_data->eventfd_ctxs[interrupt] != NULL) + eventfd_ctx_put(interrupt_data->eventfd_ctxs[interrupt]); interrupt_data->eventfd_ctxs[interrupt] = ctx; + write_unlock_irqrestore(&interrupt_data->eventfd_ctx_lock, flags); return 0; } int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data, int interrupt) { + ulong flags; + if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; + /* Put the old eventfd ctx before clearing, else we leak the ref. */ + write_lock_irqsave(&interrupt_data->eventfd_ctx_lock, flags); + if (interrupt_data->eventfd_ctxs[interrupt] != NULL) + eventfd_ctx_put(interrupt_data->eventfd_ctxs[interrupt]); interrupt_data->eventfd_ctxs[interrupt] = NULL; + write_unlock_irqrestore(&interrupt_data->eventfd_ctx_lock, flags); return 0; } From fbe01658e7baad1cc57087911a6e6cd7930b879a Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Fri, 7 Jun 2019 13:23:49 -0700 Subject: [PATCH 168/209] staging: gasket: Add performance scaling based on temperature Perform periodic polling of apex temperature and scale down its frequency if temperature is above one of defined trip points. Added sysfs nodes to control related settings on per device basis. Added module parameters to set allow setting default value for all devices in system. Signed-off-by: Leonid Lobachev Change-Id: I5065282cba9f0888abd4086601bda89174f24705 --- apex_driver.c | 203 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 200 insertions(+), 3 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index d1b823e..6f5cafe 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -5,6 +5,7 @@ * Copyright (C) 2018 Google, Inc. */ +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include "apex.h" @@ -62,6 +64,17 @@ /* Wait 100 ms between checks. Total 12 sec wait maximum. */ #define APEX_RESET_DELAY 100 +/* Interval between temperature polls, 0 disables polling */ +#define DEFAULT_APEX_TEMP_POLL_INTERVAL 0 + +/* apex device private data */ +struct apex_dev { + struct gasket_dev *gasket_dev_ptr; + struct delayed_work check_temperature_work; + u32 adc_trip_points[3]; + atomic_t temp_poll_interval; +}; + /* Enumeration of the supported sysfs entries. */ enum sysfs_attribute_type { ATTR_KERNEL_HIB_PAGE_TABLE_SIZE, @@ -72,6 +85,10 @@ enum sysfs_attribute_type { ATTR_TEMP_WARN1_EN, ATTR_TEMP_WARN2, ATTR_TEMP_WARN2_EN, + ATTR_TEMP_TRIP0, + ATTR_TEMP_TRIP1, + ATTR_TEMP_TRIP2, + ATTR_TEMP_POLL_INTERVAL, }; /* @@ -234,7 +251,6 @@ static struct gasket_interrupt_desc apex_interrupts[] = { }, }; - /* Allows device to enter power save upon driver close(). */ static int allow_power_save = 1; @@ -253,6 +269,23 @@ module_param(allow_sw_clock_gating, int, 0644); module_param(allow_hw_clock_gating, int, 0644); module_param(bypass_top_level, int, 0644); +/* Temperature points in milli C at which DFS is toggled */ +#define DEFAULT_TRIP_POINT0_TEMP 85000 +#define DEFAULT_TRIP_POINT1_TEMP 90000 +#define DEFAULT_TRIP_POINT2_TEMP 95000 + +static int trip_point0_temp = DEFAULT_TRIP_POINT0_TEMP; +static int trip_point1_temp = DEFAULT_TRIP_POINT1_TEMP; +static int trip_point2_temp = DEFAULT_TRIP_POINT2_TEMP; + +module_param(trip_point0_temp, int, 0644); +module_param(trip_point1_temp, int, 0644); +module_param(trip_point2_temp, int, 0644); + +/* Temperature poll interval in ms */ +static int temp_poll_interval = DEFAULT_APEX_TEMP_POLL_INTERVAL; +module_param(temp_poll_interval, int, 0644); + /* Check the device status registers and return device status ALIVE or DEAD. */ static int apex_get_status(struct gasket_dev *gasket_dev) { @@ -589,6 +622,7 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, int ret; unsigned value; struct gasket_dev *gasket_dev; + struct apex_dev *apex_dev; struct gasket_sysfs_attribute *gasket_attr; enum sysfs_attribute_type type; @@ -598,6 +632,13 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, return -ENODEV; } + if (!gasket_dev->pci_dev || + !(apex_dev = pci_get_drvdata(gasket_dev->pci_dev))) { + dev_err(device, "Can't find apex_dev data\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { dev_err(device, "No Apex device sysfs attr data found\n"); @@ -650,6 +691,22 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, APEX_BAR2_REG_OMC0_D8); ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); break; + case ATTR_TEMP_TRIP0: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->adc_trip_points[0])); + break; + case ATTR_TEMP_TRIP1: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->adc_trip_points[1])); + break; + case ATTR_TEMP_TRIP2: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->adc_trip_points[2])); + break; + case ATTR_TEMP_POLL_INTERVAL: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + atomic_read(&apex_dev->temp_poll_interval)); + break; default: dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", attr->attr.name); @@ -668,6 +725,7 @@ static ssize_t sysfs_store(struct device *device, struct device_attribute *attr, { int ret = count, value; struct gasket_dev *gasket_dev; + struct apex_dev *apex_dev; struct gasket_sysfs_attribute *gasket_attr; enum sysfs_attribute_type type; @@ -680,6 +738,13 @@ static ssize_t sysfs_store(struct device *device, struct device_attribute *attr, return -ENODEV; } + if (!gasket_dev->pci_dev || + !(apex_dev = pci_get_drvdata(gasket_dev->pci_dev))) { + dev_err(device, "Can't find apex_dev data\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { dev_err(device, "No Apex device sysfs attr data found\n"); @@ -713,6 +778,34 @@ static ssize_t sysfs_store(struct device *device, struct device_attribute *attr, APEX_BAR2_REG_OMC0_D8, value, 1, 31); break; + case ATTR_TEMP_TRIP0: + value = millic_to_adc(value); + /* Note: that adc values should be in descending order */ + if (value >= apex_dev->adc_trip_points[1]) { + apex_dev->adc_trip_points[0] = value; + } else ret = -EINVAL; + break; + case ATTR_TEMP_TRIP1: + value = millic_to_adc(value); + if (value <= apex_dev->adc_trip_points[0] && + value >= apex_dev->adc_trip_points[2]) { + apex_dev->adc_trip_points[1] = value; + } else ret = -EINVAL; + break; + case ATTR_TEMP_TRIP2: + value = millic_to_adc(value); + if (value <= apex_dev->adc_trip_points[1]) { + apex_dev->adc_trip_points[2] = value; + } else ret = -EINVAL; + break; + case ATTR_TEMP_POLL_INTERVAL: + cancel_delayed_work_sync(&apex_dev->check_temperature_work); + atomic_set(&apex_dev->temp_poll_interval, value); + if (value > 0) + schedule_delayed_work(&apex_dev->check_temperature_work, + msecs_to_jiffies(value)); + + break; default: dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", attr->attr.name); @@ -739,9 +832,84 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { GASKET_SYSFS_RW(temp_warn2, sysfs_show, sysfs_store, ATTR_TEMP_WARN2), GASKET_SYSFS_RW(temp_warn2_en, sysfs_show, sysfs_store, ATTR_TEMP_WARN2_EN), + GASKET_SYSFS_RW(trip_point0_temp, sysfs_show, sysfs_store, + ATTR_TEMP_TRIP0), + GASKET_SYSFS_RW(trip_point1_temp, sysfs_show, sysfs_store, + ATTR_TEMP_TRIP1), + GASKET_SYSFS_RW(trip_point2_temp, sysfs_show, sysfs_store, + ATTR_TEMP_TRIP2), + GASKET_SYSFS_RW(temp_poll_interval, sysfs_show, sysfs_store, + ATTR_TEMP_POLL_INTERVAL), GASKET_END_OF_ATTR_ARRAY }; +static void apply_module_params(struct apex_dev *apex_dev) { + kernel_param_lock(THIS_MODULE); + + /* use defaults if trip point temperatures are not in ascending order */ + if (trip_point0_temp > trip_point1_temp || + trip_point1_temp > trip_point2_temp) { + dev_warn(apex_dev->gasket_dev_ptr->dev, + "Invalid module parameters for temperature trip points" + ", using defaults\n"); + trip_point0_temp = DEFAULT_TRIP_POINT0_TEMP; + trip_point1_temp = DEFAULT_TRIP_POINT1_TEMP; + trip_point2_temp = DEFAULT_TRIP_POINT2_TEMP; + } + + apex_dev->adc_trip_points[0] = millic_to_adc(trip_point0_temp); + apex_dev->adc_trip_points[1] = millic_to_adc(trip_point1_temp); + apex_dev->adc_trip_points[2] = millic_to_adc(trip_point2_temp); + atomic_set(&apex_dev->temp_poll_interval, temp_poll_interval); + + kernel_param_unlock(THIS_MODULE); +} + +static void check_temperature_work_handler(struct work_struct *work) { + int i; + u32 adc_temp, clk_div, tmp; + const u32 mask = ((1 << 2) - 1) << 28; + struct apex_dev *apex_dev = + container_of(work, struct apex_dev, + check_temperature_work.work); + struct gasket_dev *gasket_dev = apex_dev->gasket_dev_ptr; + + mutex_lock(&gasket_dev->mutex); + + /* Read current temperature */ + adc_temp = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC); + adc_temp = (adc_temp >> 16) & ((1 << 10) - 1); + + /* Find closest trip point + Note: that adc values are in descending order */ + for (i = ARRAY_SIZE(apex_dev->adc_trip_points) - 1; i >= 0; --i) { + if (adc_temp <= apex_dev->adc_trip_points[i]) + break; + } + /* Compute divider value and shift into appropriate bit location */ + clk_div = (i + 1) << 28; + + /* Modify gcb clk divider if it's different from current one */ + tmp = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + if (clk_div != (tmp & mask)) { + tmp = (tmp & ~mask) | clk_div; + gasket_dev_write_32(gasket_dev, tmp, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + dev_warn(gasket_dev->dev, + "Apex performance %sthrottled due to temperature\n", + i == -1 ? "not " : ""); + } + + mutex_unlock(&gasket_dev->mutex); + + int temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); + if (temp_poll_interval > 0) + schedule_delayed_work(&apex_dev->check_temperature_work, + msecs_to_jiffies(temp_poll_interval)); +} + /* On device open, perform a core reinit reset. */ static int apex_device_open_cb(struct gasket_dev *gasket_dev) { @@ -766,6 +934,7 @@ static int apex_pci_probe(struct pci_dev *pci_dev, ulong page_table_ready, msix_table_ready; int retries = 0; struct gasket_dev *gasket_dev; + struct apex_dev *apex_dev; ret = pci_enable_device(pci_dev); #ifdef MODULE @@ -789,7 +958,18 @@ static int apex_pci_probe(struct pci_dev *pci_dev, return ret; } - pci_set_drvdata(pci_dev, gasket_dev); + apex_dev = kzalloc(sizeof(*apex_dev), GFP_KERNEL); + if (!apex_dev) { + dev_err(&pci_dev->dev, "no memory for device\n"); + ret = -ENOMEM; + goto remove_device; + } + + INIT_DELAYED_WORK(&apex_dev->check_temperature_work, + check_temperature_work_handler); + apex_dev->gasket_dev_ptr = gasket_dev; + apply_module_params(apex_dev); + pci_set_drvdata(pci_dev, apex_dev); apex_reset(gasket_dev); while (retries < APEX_RESET_RETRY) { @@ -843,19 +1023,36 @@ static int apex_pci_probe(struct pci_dev *pci_dev, if (allow_power_save) apex_enter_reset(gasket_dev); + /* Enable thermal polling */ + int temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); + if (temp_poll_interval > 0) + schedule_delayed_work(&apex_dev->check_temperature_work, + msecs_to_jiffies(temp_poll_interval)); return 0; remove_device: gasket_pci_remove_device(pci_dev); pci_disable_device(pci_dev); + kfree(apex_dev); return ret; } static void apex_pci_remove(struct pci_dev *pci_dev) { - struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev); + struct apex_dev *apex_dev = pci_get_drvdata(pci_dev); + struct gasket_dev *gasket_dev; + + if (!apex_dev) { + dev_err(&pci_dev->dev, "NULL apex_dev\n"); + goto remove_device; + } + gasket_dev = apex_dev->gasket_dev_ptr; + + cancel_delayed_work_sync(&apex_dev->check_temperature_work); + kfree(apex_dev); gasket_disable_device(gasket_dev); +remove_device: gasket_pci_remove_device(pci_dev); pci_disable_device(pci_dev); } From 4856195932d601c7b5cd1ee72bab3d48fe783245 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 10 Jun 2019 15:41:17 -0700 Subject: [PATCH 169/209] staging: gasket: Add default values for HW temperature monitor Add module parameters to control default values for HW temperature monitoring module. Signed-off-by: Leonid Lobachev Change-Id: I69f7cd9c3c0f55a3a36efba35f48a5da37b3b89e --- apex_driver.c | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 6f5cafe..4476b67 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -282,6 +282,21 @@ module_param(trip_point0_temp, int, 0644); module_param(trip_point1_temp, int, 0644); module_param(trip_point2_temp, int, 0644); +/* Hardware monitored temperature trip points in milli C + Apex chip drives INTR line when reaching hw_temp_warn1 temperature, + and SD_ALARM line when reaching hw_temp_warn2 if corresponding + hw_temp_warn*_en is set to true. + */ +static int hw_temp_warn1 = 100000; +static int hw_temp_warn2 = 110000; +static bool hw_temp_warn1_en = false; +static bool hw_temp_warn2_en = false; + +module_param(hw_temp_warn1, int, 0644); +module_param(hw_temp_warn2, int, 0644); +module_param(hw_temp_warn1_en, bool, 0644); +module_param(hw_temp_warn2_en, bool, 0644); + /* Temperature poll interval in ms */ static int temp_poll_interval = DEFAULT_APEX_TEMP_POLL_INTERVAL; module_param(temp_poll_interval, int, 0644); @@ -826,11 +841,13 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), GASKET_SYSFS_RO(temp, sysfs_show, ATTR_TEMP), - GASKET_SYSFS_RW(temp_warn1, sysfs_show, sysfs_store, ATTR_TEMP_WARN1), - GASKET_SYSFS_RW(temp_warn1_en, sysfs_show, sysfs_store, + GASKET_SYSFS_RW(hw_temp_warn1, sysfs_show, sysfs_store, + ATTR_TEMP_WARN1), + GASKET_SYSFS_RW(hw_temp_warn1_en, sysfs_show, sysfs_store, ATTR_TEMP_WARN1_EN), - GASKET_SYSFS_RW(temp_warn2, sysfs_show, sysfs_store, ATTR_TEMP_WARN2), - GASKET_SYSFS_RW(temp_warn2_en, sysfs_show, sysfs_store, + GASKET_SYSFS_RW(hw_temp_warn2, sysfs_show, sysfs_store, + ATTR_TEMP_WARN2), + GASKET_SYSFS_RW(hw_temp_warn2_en, sysfs_show, sysfs_store, ATTR_TEMP_WARN2_EN), GASKET_SYSFS_RW(trip_point0_temp, sysfs_show, sysfs_store, ATTR_TEMP_TRIP0), @@ -862,6 +879,22 @@ static void apply_module_params(struct apex_dev *apex_dev) { apex_dev->adc_trip_points[2] = millic_to_adc(trip_point2_temp); atomic_set(&apex_dev->temp_poll_interval, temp_poll_interval); + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, + millic_to_adc(hw_temp_warn1), 10, 16); + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, + millic_to_adc(hw_temp_warn2), 10, 16); + if (hw_temp_warn1_en) + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, + APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, 1, 1, 31); + + if (hw_temp_warn2_en) + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, + APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, 1, 1, 31); + kernel_param_unlock(THIS_MODULE); } From 83d87f28f847627e7d996ff4a5def9ecc98da397 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Tue, 11 Jun 2019 11:40:27 -0700 Subject: [PATCH 170/209] staging: gasket: Enable DFS and thermal shutdown by default Change-Id: I6f9f910f3fa7fdefa1ed3a9d81bb35fb69b68fbb Signed-off-by: Leonid Lobachev --- apex_driver.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 4476b67..7389cf8 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -65,7 +65,7 @@ #define APEX_RESET_DELAY 100 /* Interval between temperature polls, 0 disables polling */ -#define DEFAULT_APEX_TEMP_POLL_INTERVAL 0 +#define DEFAULT_APEX_TEMP_POLL_INTERVAL 5000 /* apex device private data */ struct apex_dev { @@ -288,9 +288,9 @@ module_param(trip_point2_temp, int, 0644); hw_temp_warn*_en is set to true. */ static int hw_temp_warn1 = 100000; -static int hw_temp_warn2 = 110000; +static int hw_temp_warn2 = 100000; static bool hw_temp_warn1_en = false; -static bool hw_temp_warn2_en = false; +static bool hw_temp_warn2_en = true; module_param(hw_temp_warn1, int, 0644); module_param(hw_temp_warn2, int, 0644); From ad4c9a589344c072cda469995017ccb003d58a53 Mon Sep 17 00:00:00 2001 From: Nick Ewalt Date: Thu, 11 Oct 2018 14:56:39 -0700 Subject: [PATCH 171/209] staging: gasket: fix free in gasket_free_coherent_memory_all Actually free gasket_coherent_page_entries and just call gasket_free_coherent_memory_all from gasket_free_coherent_memory. Change-Id: I1d016abd5f66e0620a37f6851e43fcfd6c96868a Signed-off-by: Nick Ewalt --- gasket_page_table.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 3aff2aa..bf5c8ff 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1344,15 +1344,8 @@ int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, if (driver_desc->coherent_buffer_description.base != dma_address) return -EADDRNOTAVAIL; - if (gasket_dev->coherent_buffer.length_bytes) { - dma_free_coherent(gasket_get_device(gasket_dev), - gasket_dev->coherent_buffer.length_bytes, - gasket_dev->coherent_buffer.virt_base, - gasket_dev->coherent_buffer.phys_base); - gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = NULL; - gasket_dev->coherent_buffer.phys_base = 0; - } + gasket_free_coherent_memory_all(gasket_dev, index); + return 0; } @@ -1372,4 +1365,8 @@ void gasket_free_coherent_memory_all( gasket_dev->coherent_buffer.virt_base = NULL; gasket_dev->coherent_buffer.phys_base = 0; } + + kfree(gasket_dev->page_table[index]->coherent_pages); + gasket_dev->page_table[index]->coherent_pages = NULL; + gasket_dev->page_table[index]->num_coherent_pages = 0; } From 5862a44ee2cc9f204cdc17d043912d5f96248a90 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 16 Aug 2018 05:14:04 -0700 Subject: [PATCH 172/209] staging: gasket: page table: use GFP_KERNEL for dma_alloc_coherent Flags should be specified for dma_alloc_coherent() call. Use GFP_KERNEL, it's fine to sleep here. Change-Id: Iec4ad9550e8829b67e99b22246e40e0b21600c54 Signed-off-by: Todd Poynor --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index bf5c8ff..42d479a 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1287,7 +1287,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, return -EINVAL; mem = dma_alloc_coherent(gasket_get_device(gasket_dev), - num_pages * PAGE_SIZE, &handle, 0); + num_pages * PAGE_SIZE, &handle, GFP_KERNEL); if (!mem) goto nomem; From 7d66238c086441b7ce7da5d78b705d112fc6c5c1 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:44:07 -0700 Subject: [PATCH 173/209] staging: gasket: interrupt: remove PCI-MSIX-specific status check Devices not using MSIX don't use the msix_initialized field, don't require it to be set in the interrupt system status check. The general check for interrupts configured that follows can cover both MSIX and device-managed interrupts. Change-Id: I83a86cb4db7cea30c30377dd82b033ef372d69c0 Signed-off-by: Todd Poynor --- gasket_interrupt.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 3cf7235..9d0ad2c 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -488,11 +488,6 @@ int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) return GASKET_STATUS_DEAD; } - if (!gasket_dev->interrupt_data->msix_configured) { - dev_dbg(gasket_dev->dev, "Interrupt not initialized\n"); - return GASKET_STATUS_LAMED; - } - if (gasket_dev->interrupt_data->num_configured != gasket_dev->interrupt_data->num_interrupts) { dev_dbg(gasket_dev->dev, From bc67c21fb1e6d386270138074dd9cac53c8a3d9e Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:47:33 -0700 Subject: [PATCH 174/209] staging: gasket: core: debug log updates Add debug logs for device enable/disable events, remove logs for callbacks (the called functions can generate their own logs if needed). Change-Id: I861077c3210452d9ac6606e973bac5c14eeef859 Signed-off-by: Todd Poynor --- gasket_core.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index fe55ab0..1cbae4d 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -109,8 +109,6 @@ check_and_invoke_callback(struct gasket_dev *gasket_dev, { int ret = 0; - dev_dbg(gasket_dev->dev, "check_and_invoke_callback %p\n", - cb_function); if (cb_function) { mutex_lock(&gasket_dev->mutex); ret = cb_function(gasket_dev); @@ -126,11 +124,8 @@ gasket_check_and_invoke_callback_nolock(struct gasket_dev *gasket_dev, { int ret = 0; - if (cb_function) { - dev_dbg(gasket_dev->dev, - "Invoking device-specific callback.\n"); + if (cb_function) ret = cb_function(gasket_dev); - } return ret; } @@ -633,6 +628,7 @@ void gasket_disable_device(struct gasket_dev *gasket_dev) gasket_dev->internal_desc->driver_desc; int i; + dev_dbg(gasket_dev->dev, "disabling device\n"); /* Only delete the device if it has been successfully added. */ if (gasket_dev->dev_info.cdev_added) cdev_del(&gasket_dev->dev_info.cdev); @@ -1358,6 +1354,7 @@ int gasket_enable_device(struct gasket_dev *gasket_dev) const struct gasket_driver_desc *driver_desc = gasket_dev->internal_desc->driver_desc; + dev_dbg(gasket_dev->dev, "enabling device\n"); ret = gasket_interrupt_init(gasket_dev); if (ret) { dev_err(gasket_dev->dev, From 71591a83add8dd8de04f51abdd68770ff51f57ee Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:49:33 -0700 Subject: [PATCH 175/209] staging: gasket: page table: return valid error code on map fail Return -EINVAL on mapping failures, instead of -1, which triggers a checkpatch error. Change-Id: I91a62a72caa5ca09e07bd4897e72f4919495c040 Signed-off-by: Todd Poynor --- gasket_page_table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 42d479a..51497d0 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -500,7 +500,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, (unsigned long long)ptes[i].dma_addr, (void *)page_to_pfn(page), (void *)page_to_phys(page)); - return -1; + return -EINVAL; } } @@ -1147,7 +1147,7 @@ fail: *ppage = NULL; *poffset = 0; mutex_unlock(&pg_tbl->mutex); - return -1; + return -EINVAL; } /* See gasket_page_table.h for description. */ From 3e21ff65d68ab02ecb926383057ab8d537018350 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:50:56 -0700 Subject: [PATCH 176/209] staging: gasket: page table: remove dead code in coherent mem alloc gasket_alloc_coherent_memory() has some unnecessary code related to out of memory checking that will never hit the condition checked, remove. Change-Id: Ifb09e7ff31d8ae41a486639a2abc6ce26c4d85ef Signed-off-by: Todd Poynor --- gasket_page_table.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 51497d0..a79d9b0 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1299,7 +1299,6 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, GFP_KERNEL); if (!gasket_dev->page_table[index]->coherent_pages) goto nomem; - *dma_address = 0; gasket_dev->coherent_buffer.length_bytes = PAGE_SIZE * (num_pages); @@ -1314,15 +1313,12 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, (dma_addr_t)mem + j * PAGE_SIZE; } - if (*dma_address == 0) - goto nomem; return 0; nomem: - if (mem) { + if (mem) dma_free_coherent(gasket_get_device(gasket_dev), num_pages * PAGE_SIZE, mem, handle); - } kfree(gasket_dev->page_table[index]->coherent_pages); gasket_dev->page_table[index]->coherent_pages = NULL; From 06938d183cc5c92e2e81ca518d3ff21765866170 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:53:12 -0700 Subject: [PATCH 177/209] staging: gasket: page_table: fix dma_data_direction Extended page tables can be mapped as DMA_TO_DEVICE. Calls to dma_unmap_page should match corresponding dma_map_page. Change-Id: I2d09fd5d66c364e190b073fabe089ce5eaf426f6 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index a79d9b0..5206393 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -320,7 +320,7 @@ static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl, if (pte->dma_addr) dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, - DMA_BIDIRECTIONAL); + DMA_TO_DEVICE); vfree(pte->sublevel); @@ -612,7 +612,7 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, if (ptes[i].status == PTE_INUSE) { if (ptes[i].dma_addr) { dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, - PAGE_SIZE, DMA_FROM_DEVICE); + PAGE_SIZE, DMA_BIDIRECTIONAL); } if (gasket_release_page(ptes[i].page)) --pg_tbl->num_active_pages; @@ -894,7 +894,7 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, /* Map the page into DMA space. */ pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); + DMA_TO_DEVICE); /* make the addresses available to the device */ dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; From 7e99cc991ac840a99eb42174d1eb47da50069a92 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:55:10 -0700 Subject: [PATCH 178/209] staging: gasket: page_table: rearrange gasket_page_table_entry Rearrange gasket_page_table entry to reduce padding slop. Change-Id: I1dab69344bb1974de0d67028e6f638ec5a849374 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 5206393..3fd758b 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -103,12 +103,6 @@ struct gasket_page_table_entry { /* The status of this entry/slot: free or in use. */ enum pte_status status; - /* Address of the page in DMA space. */ - dma_addr_t dma_addr; - - /* Linux page descriptor for the page described by this structure. */ - struct page *page; - /* * Index for alignment into host vaddrs. * When a user specifies a host address for a mapping, that address may @@ -119,6 +113,12 @@ struct gasket_page_table_entry { */ int offset; + /* Address of the page in DMA space. */ + dma_addr_t dma_addr; + + /* Linux page descriptor for the page described by this structure. */ + struct page *page; + /* * If this is an extended and first-level entry, sublevel points * to the second-level entries underneath this entry. From 513c6336ef0c3be891241026a54aa4165cbe4901 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 11:56:47 -0700 Subject: [PATCH 179/209] staging: gasket: page_table: remove unnecessary PTE status set to free Remove unnecessary ptes[i].status update in gasket_perform_unmapping. The vlaue will be cleared in the following memset. Change-Id: I5aeafa1085203b5dc24b3303037ece1f37acd6c6 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 1 - 1 file changed, 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 3fd758b..cbff439 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -617,7 +617,6 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, if (gasket_release_page(ptes[i].page)) --pg_tbl->num_active_pages; } - ptes[i].status = PTE_FREE; /* and clear the PTE. */ memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); From bcf565dc8194efda906f871c0f7de057b3a48d58 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 13:53:42 -0700 Subject: [PATCH 180/209] staging: gasket: page_table: add mapping flags This allows for more precise dma_direction in the dma_map_page requests. Also leaves room for adding more flags later. Change-Id: I46feecc2638b7ad0d34548cc10837549a3928011 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket.h | 33 +++++++++++++++++ gasket_ioctl.c | 70 +++++++++++++++++++++++++++--------- gasket_page_table.c | 87 +++++++++++++++++++++++++++++---------------- gasket_page_table.h | 4 ++- 4 files changed, 145 insertions(+), 49 deletions(-) diff --git a/gasket.h b/gasket.h index a0f065c..93e7af1 100644 --- a/gasket.h +++ b/gasket.h @@ -37,6 +37,31 @@ struct gasket_page_table_ioctl { u64 device_address; }; +/* + * Structure for ioctl mapping buffers with flags when using the Gasket + * page_table module. + */ +struct gasket_page_table_ioctl_flags { + struct gasket_page_table_ioctl base; + /* + * Flags indicating status and attribute requests from the host. + * NOTE: STATUS bit does not need to be set in this request. + * Set RESERVED bits to 0 to ensure backwards compatibility. + * + * Bitfields: + * [0] - STATUS: indicates if this entry/slot is free + * 0 = PTE_FREE + * 1 = PTE_INUSE + * [2:1] - DMA_DIRECTION: dma_data_direction requested by host + * 00 = DMA_BIDIRECTIONAL + * 01 = DMA_TO_DEVICE + * 10 = DMA_FROM_DEVICE + * 11 = DMA_NONE + * [31:3] - RESERVED + */ + u32 flags; +}; + /* * Common structure for ioctls mapping and unmapping buffers when using the * Gasket page_table module. @@ -119,4 +144,12 @@ struct gasket_coherent_alloc_config_ioctl { #define GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR \ _IOWR(GASKET_IOCTL_BASE, 11, struct gasket_coherent_alloc_config_ioctl) +/* + * Tells the kernel to map size bytes at host_address to device_address in + * page_table_index page table. Passes flags to indicate additional attribute + * requests for the mapped memory. + */ +#define GASKET_IOCTL_MAP_BUFFER_FLAGS \ + _IOW(GASKET_IOCTL_BASE, 12, struct gasket_page_table_ioctl_flags) + #endif /* __GASKET_H__ */ diff --git a/gasket_ioctl.c b/gasket_ioctl.c index d32bce9..4696007 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -20,6 +20,7 @@ #define trace_gasket_ioctl_integer_data(x) #define trace_gasket_ioctl_eventfd_data(x, ...) #define trace_gasket_ioctl_page_table_data(x, ...) +#define trace_gasket_ioctl_page_table_flags_data(x, ...) #define trace_gasket_ioctl_config_coherent_allocator(x, ...) #endif @@ -130,29 +131,59 @@ static int gasket_partition_page_table( } /* Map a userspace buffer to a device virtual address. */ +static int gasket_map_buffers_common(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl_flags + *pibuf) +{ + if (pibuf->base.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + if (gasket_page_table_are_addrs_bad(gasket_dev->page_table[pibuf->base.page_table_index], + pibuf->base.host_address, + pibuf->base.device_address, + pibuf->base.size)) + return -EINVAL; + + return gasket_page_table_map(gasket_dev->page_table[pibuf->base.page_table_index], + pibuf->base.host_address, + pibuf->base.device_address, + pibuf->base.size / PAGE_SIZE, + pibuf->flags); +} + static int gasket_map_buffers(struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) { - struct gasket_page_table_ioctl ibuf; + struct gasket_page_table_ioctl_flags ibuf; + + if (copy_from_user(&ibuf.base, argp, sizeof(struct gasket_page_table_ioctl))) + return -EFAULT; + + ibuf.flags = 0; + + trace_gasket_ioctl_page_table_data(ibuf.base.page_table_index, + ibuf.base.size, + ibuf.base.host_address, + ibuf.base.device_address); - if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) - return -EFAULT; + return gasket_map_buffers_common(gasket_dev, &ibuf); +} - trace_gasket_ioctl_page_table_data(ibuf.page_table_index, ibuf.size, - ibuf.host_address, - ibuf.device_address); +static int gasket_map_buffers_flags(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl_flags __user *argp) +{ + struct gasket_page_table_ioctl_flags ibuf; - if (ibuf.page_table_index >= gasket_dev->num_page_tables) - return -EFAULT; - - if (gasket_page_table_are_addrs_bad(gasket_dev->page_table[ibuf.page_table_index], - ibuf.host_address, - ibuf.device_address, ibuf.size)) - return -EINVAL; - - return gasket_page_table_map(gasket_dev->page_table[ibuf.page_table_index], - ibuf.host_address, ibuf.device_address, - ibuf.size / PAGE_SIZE); + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl_flags))) + return -EFAULT; + + trace_gasket_ioctl_page_table_flags_data(ibuf.base.page_table_index, + ibuf.base.size, + ibuf.base.host_address, + ibuf.base.device_address, + ibuf.flags); + + return gasket_map_buffers_common(gasket_dev, &ibuf); } /* Unmap a userspace buffer from a device virtual address. */ @@ -258,6 +289,7 @@ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) return alive && write; case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_MAP_BUFFER_FLAGS: case GASKET_IOCTL_UNMAP_BUFFER: return alive && write; @@ -342,6 +374,9 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) case GASKET_IOCTL_MAP_BUFFER: retval = gasket_map_buffers(gasket_dev, argp); break; + case GASKET_IOCTL_MAP_BUFFER_FLAGS: + retval = gasket_map_buffers_flags(gasket_dev, argp); + break; case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: retval = gasket_config_coherent_allocator(gasket_dev, argp); break; @@ -387,6 +422,7 @@ long gasket_is_supported_ioctl(uint cmd) case GASKET_IOCTL_PAGE_TABLE_SIZE: case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_MAP_BUFFER_FLAGS: case GASKET_IOCTL_UNMAP_BUFFER: case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: diff --git a/gasket_page_table.c b/gasket_page_table.c index cbff439..33672ee 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -79,6 +79,19 @@ */ #define GASKET_EXTENDED_LVL1_SHIFT 12 +/* + * Utilities for accessing flags bitfields. + */ +#define MASK(field) (((1u << field##_WIDTH) - 1) << field##_SHIFT) +#define GET(field, flags) (((flags) & MASK(field)) >> field##_SHIFT) +#define SET(field, flags, val) (((flags) & ~MASK(field)) | ((val) << field##_SHIFT)) + +#define FLAGS_STATUS_SHIFT 0 +#define FLAGS_STATUS_WIDTH 1 + +#define FLAGS_DMA_DIRECTION_SHIFT 1 +#define FLAGS_DMA_DIRECTION_WIDTH 2 + /* Type declarations */ /* Valid states for a struct gasket_page_table_entry. */ enum pte_status { @@ -100,8 +113,12 @@ enum pte_status { * to the actual page mapped and described by this structure. */ struct gasket_page_table_entry { - /* The status of this entry/slot: free or in use. */ - enum pte_status status; + /* + * Internal structure matches gasket_page_table_ioctl_flags.flags. + * NOTE: All fields should have a default value of 0. This ensures that + * the kernel will be backwards compatible with old drivers. + */ + u32 flags; /* * Index for alignment into host vaddrs. @@ -297,7 +314,7 @@ static bool gasket_is_pte_range_free(struct gasket_page_table_entry *ptes, int i; for (i = 0; i < num_entries; i++) { - if (ptes[i].status != PTE_FREE) + if (GET(FLAGS_STATUS, ptes[i].flags) != PTE_FREE) return false; } @@ -313,7 +330,7 @@ static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl, u64 __iomem *slot) { /* Release the page table from the driver */ - pte->status = PTE_FREE; + pte->flags = SET(FLAGS_STATUS, pte->flags, PTE_FREE); /* Release the page table from the device */ writeq(0, slot); @@ -347,7 +364,7 @@ gasket_page_table_garbage_collect_nolock(struct gasket_page_table *pg_tbl) slot = pg_tbl->base_slot + pg_tbl->num_simple_entries; pte < pg_tbl->entries + pg_tbl->config.total_entries; pte++, slot++) { - if (pte->status == PTE_INUSE) { + if (GET(FLAGS_STATUS, pte->flags) == PTE_INUSE) { if (gasket_is_pte_range_free(pte->sublevel, GASKET_PAGES_PER_SUBTABLE)) gasket_free_extended_subtable(pg_tbl, pte, @@ -396,7 +413,7 @@ int gasket_page_table_partition(struct gasket_page_table *pg_tbl, start = min(pg_tbl->num_simple_entries, num_simple_entries); for (i = start; i < pg_tbl->config.total_entries; i++) { - if (pg_tbl->entries[i].status != PTE_FREE) { + if (GET(FLAGS_STATUS, pg_tbl->entries[i].flags) != PTE_FREE) { dev_err(pg_tbl->device, "entry %d is not free\n", i); mutex_unlock(&pg_tbl->mutex); return -EBUSY; @@ -444,7 +461,8 @@ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, u64 __iomem *slots, ulong host_addr, - uint num_pages, int is_simple_mapping) + uint num_pages, u32 flags, + int is_simple_mapping) { int ret; ulong offset; @@ -453,6 +471,12 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ulong page_addr; int i; + if (GET(FLAGS_DMA_DIRECTION, flags) == DMA_NONE) { + dev_err(pg_tbl->device, "invalid DMA direction flags=0x%lx\n", + (unsigned long)flags); + return -EINVAL; + } + for (i = 0; i < num_pages; i++) { page_addr = host_addr + i * PAGE_SIZE; offset = page_addr & (PAGE_SIZE - 1); @@ -482,9 +506,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ptes[i].offset = offset; /* Map the page into DMA space. */ - ptes[i].dma_addr = - dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); + ptes[i].dma_addr = dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, + GET(FLAGS_DMA_DIRECTION, flags)); dev_dbg(pg_tbl->device, "%s i %d pte %p pfn %p -> mapped %llx\n", __func__, i, &ptes[i], @@ -518,7 +541,9 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, (void *)&((u64 __force *)slots)[i], sizeof(u64), DMA_TO_DEVICE); } - ptes[i].status = PTE_INUSE; + + /* Set PTE flags equal to flags param with STATUS=PTE_INUSE. */ + ptes[i].flags = SET(FLAGS_STATUS, flags, PTE_INUSE); } return 0; } @@ -600,7 +625,8 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, */ for (i = 0; i < num_pages; i++) { /* release the address from the device, */ - if (is_simple_mapping || ptes[i].status == PTE_INUSE) { + if (is_simple_mapping || + GET(FLAGS_STATUS, ptes[i].flags) == PTE_INUSE) { writeq(0, &slots[i]); } else { ((u64 __force *)slots)[i] = 0; @@ -609,11 +635,10 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, } /* release the address from the driver, */ - if (ptes[i].status == PTE_INUSE) { + if (GET(FLAGS_STATUS, ptes[i].flags) == PTE_INUSE) { if (ptes[i].dma_addr) { - dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, - PAGE_SIZE, DMA_BIDIRECTIONAL); - } + dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, PAGE_SIZE, + GET(FLAGS_DMA_DIRECTION, ptes[i].flags)); } if (gasket_release_page(ptes[i].page)) --pg_tbl->num_active_pages; } @@ -656,7 +681,7 @@ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, /* TODO: Add check to ensure pte remains valid? */ len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); - if (pte->status == PTE_INUSE) { + if (GET(FLAGS_STATUS, pte->flags) == PTE_INUSE) { slot_base = (u64 __iomem *)(page_address(pte->page) + pte->offset); gasket_perform_unmapping(pg_tbl, @@ -835,7 +860,7 @@ static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, */ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) + uint num_pages, u32 flags) { int ret; uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); @@ -850,7 +875,7 @@ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx, pg_tbl->base_slot + slot_idx, host_addr, - num_pages, 1); + num_pages, flags, 1); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); @@ -899,7 +924,7 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; writeq(dma_addr, slot); - pte->status = PTE_INUSE; + pte->flags = SET(FLAGS_STATUS, pte->flags, PTE_INUSE); return 0; } @@ -937,7 +962,7 @@ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, len = min(remain, GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx); - if (pte->status == PTE_FREE) { + if (GET(FLAGS_STATUS, pte->flags) == PTE_FREE) { ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); if (ret) { dev_err(pg_tbl->device, @@ -965,7 +990,7 @@ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, */ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, - uint num_pages) + uint num_pages, u32 flags) { int ret; ulong dev_addr_end; @@ -999,7 +1024,7 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, (u64 __iomem *)(page_address(pte->page) + pte->offset); ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, host_addr, - len, 0); + len, flags, 0); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); @@ -1024,7 +1049,7 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, * The page table mutex is held for the entire operation. */ int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, - ulong dev_addr, uint num_pages) + ulong dev_addr, uint num_pages, u32 flags) { int ret; @@ -1035,18 +1060,18 @@ int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, if (gasket_addr_is_simple(pg_tbl, dev_addr)) { ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr, - num_pages); + num_pages, flags); } else { ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr, - num_pages); + num_pages, flags); } mutex_unlock(&pg_tbl->mutex); dev_dbg(pg_tbl->device, - "%s done: ha %llx daddr %llx num %d, ret %d\n", + "%s done: ha %llx daddr %llx num %d, flags %x ret %d\n", __func__, (unsigned long long)host_addr, - (unsigned long long)dev_addr, num_pages, ret); + (unsigned long long)dev_addr, num_pages, flags, ret); return ret; } EXPORT_SYMBOL(gasket_page_table_map); @@ -1118,7 +1143,7 @@ int gasket_page_table_lookup_page( goto fail; pte = pg_tbl->entries + page_num; - if (pte->status != PTE_INUSE) + if (GET(FLAGS_STATUS, pte->flags) != PTE_INUSE) goto fail; } else { /* Find the level 0 entry, */ @@ -1127,13 +1152,13 @@ int gasket_page_table_lookup_page( goto fail; pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num; - if (pte->status != PTE_INUSE) + if (GET(FLAGS_STATUS, pte->flags) != PTE_INUSE) goto fail; /* and its contained level 1 entry. */ page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); pte = pte->sublevel + page_num; - if (pte->status != PTE_INUSE) + if (GET(FLAGS_STATUS, pte->flags) != PTE_INUSE) goto fail; } diff --git a/gasket_page_table.h b/gasket_page_table.h index 783f594..e61b07a 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -85,6 +85,8 @@ int gasket_page_table_partition(struct gasket_page_table *page_table, * @host_addr: Starting host virtual memory address of the pages. * @dev_addr: Starting device address of the pages. * @num_pages: Number of [4kB] pages to map. + * @flags: Specifies attributes to apply to the pages. + * Internal structure matches gasket_page_table_ioctl_flags.flags. * * Description: Maps the "num_pages" pages of host memory pointed to by * host_addr to the address "dev_addr" in device memory. @@ -95,7 +97,7 @@ int gasket_page_table_partition(struct gasket_page_table *page_table, * If there is an error, no pages are mapped. */ int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, - ulong dev_addr, uint num_pages); + ulong dev_addr, uint num_pages, u32 flags); /* * Un-map host pages from device memory. From 3ef19e3777edb3c5194dddf6afe3fe68293fb306 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 13:55:29 -0700 Subject: [PATCH 181/209] staging: gasket: page_table: don't unmap coherent pages Only call dma_unmap_page if there was an associated dma_map_page. Change-Id: I6234719f8ab6b1e5e069c21e2f639eee4f22b039 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 33672ee..28b0042 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -636,9 +636,10 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, /* release the address from the driver, */ if (GET(FLAGS_STATUS, ptes[i].flags) == PTE_INUSE) { - if (ptes[i].dma_addr) { + if (ptes[i].page && ptes[i].dma_addr) { dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, PAGE_SIZE, - GET(FLAGS_DMA_DIRECTION, ptes[i].flags)); } + GET(FLAGS_DMA_DIRECTION, ptes[i].flags)); + } if (gasket_release_page(ptes[i].page)) --pg_tbl->num_active_pages; } From 355e45e64b42f02857198886fd59d3a6c274503a Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 13:57:18 -0700 Subject: [PATCH 182/209] staging: gasket: page table: fixup error path allocating coherent mem Correctly clean up data structure state in gasket_alloc_coherent_memory error path, to ensure no double free on the stale pointer value. Change-Id: Ifbc426e857d6bd05a13a725f2c2fdb07dedc098c Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 28b0042..c0a8096 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1341,9 +1341,13 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, return 0; nomem: - if (mem) + if (mem) { dma_free_coherent(gasket_get_device(gasket_dev), num_pages * PAGE_SIZE, mem, handle); + gasket_dev->coherent_buffer.length_bytes = 0; + gasket_dev->coherent_buffer.virt_base = NULL; + gasket_dev->coherent_buffer.phys_base = 0; + } kfree(gasket_dev->page_table[index]->coherent_pages); gasket_dev->page_table[index]->coherent_pages = NULL; From b2c73689099687ae0d886d965572b656ae69e0a0 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:00:20 -0700 Subject: [PATCH 183/209] staging: gasket: cleanup if dma_map_page fails in gasket_perform_mapping Previously pages would have never been unmapped in this case. Change-Id: I71b6447267933865ab23a3ba3830e7ce3d3e0e57 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index c0a8096..6ae4424 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -450,6 +450,19 @@ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) return min <= host_addr && host_addr < max; } +/* Safely return a page to the OS. */ +static bool gasket_release_page(struct page *page) +{ + if (!page) + return false; + + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); + + return true; +} + /* * Get and map last level page table buffers. * @@ -523,6 +536,12 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, (unsigned long long)ptes[i].dma_addr, (void *)page_to_pfn(page), (void *)page_to_phys(page)); + + /* clean up */ + if (gasket_release_page(ptes[i].page)) + --pg_tbl->num_active_pages; + + memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); return -EINVAL; } } @@ -596,19 +615,6 @@ static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl, return 0; } -/* Safely return a page to the OS. */ -static bool gasket_release_page(struct page *page) -{ - if (!page) - return false; - - if (!PageReserved(page)) - SetPageDirty(page); - put_page(page); - - return true; -} - /* * Unmap and release mapped pages. * The page table mutex must be held by the caller. From 39d3974ac0d15a4439eac92e56b2b32e924fee84 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:01:41 -0700 Subject: [PATCH 184/209] staging: gasket: page_table: fix debug log in gasket_perform_mapping Change-Id: I841914a6d5362073d0b85732b38f52f93a3a51b3 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 6ae4424..9a239f0 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -531,7 +531,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ptes[i].dma_addr)) { dev_dbg(pg_tbl->device, "%s i %d -> fail to map page %llx " - "[pfn %p ohys %p]\n", + "[pfn %p phys %p]\n", __func__, i, (unsigned long long)ptes[i].dma_addr, (void *)page_to_pfn(page), From 79b8ea00a1138f314d8e30a48ddaa1907b40a5ce Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:02:37 -0700 Subject: [PATCH 185/209] staging: gasket: page_table: use total_entries for max ext lvl0 page idx The maximum number of entries in the page table is configurable at initialization time and should be used in gasket_extended_lvl0_page_idx. Change-Id: I39f2dba48083a3eef3951ecd9659ee9f4901275e Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 9a239f0..a29426a 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -586,7 +586,7 @@ static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, u64 dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & - ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); + (pg_tbl->config.total_entries - 1); } /* From 0963c2f012eb67527fd5caa1920b32666bf2d562 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:04:01 -0700 Subject: [PATCH 186/209] staging: gasket: page_table: fix comment in components_to_dev_address Change-Id: Id343101cfdcad2354adb1fc03699e826b14192bb Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index a29426a..1c35c0a 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -713,13 +713,13 @@ static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, * Convert (simple, page, offset) into a device address. * Examples: * Simple page 0, offset 32: - * Input (0, 0, 32), Output 0x20 + * Input (1, 0, 32), Output 0x20 * Simple page 1000, offset 511: - * Input (0, 1000, 512), Output 0x3E81FF + * Input (1, 1000, 511), Output 0x3E81FF * Extended page 0, offset 32: * Input (0, 0, 32), Output 0x8000000020 * Extended page 1000, offset 511: - * Input (1, 1000, 512), Output 0x8003E81FF + * Input (0, 1000, 511), Output 0x8003E81FF */ static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, int is_simple, uint page_index, From 85c7608274bf728839be45f91b11ed31e8371671 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:05:18 -0700 Subject: [PATCH 187/209] staging: gasket: page_table: simplify gasket_components_to_dev_address Refactor gasket_components_to_dev_address to be faster and easier to understand. The old implementation was unnecessarily complex and masked the page_index for simple addresses but not extended ones. It makes the most sense for this function to perform no such masking. Change-Id: If5f68cb6f1bf3000a5d5b77aab89cb8e391c3cf4 Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 1c35c0a..4e84864 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -725,26 +725,9 @@ static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, int is_simple, uint page_index, uint offset) { - ulong lvl0_index, lvl1_index; + ulong dev_addr = (page_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; - if (is_simple) { - /* Return simple addresses directly. */ - lvl0_index = page_index & (pg_tbl->config.total_entries - 1); - return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; - } - - /* - * This could be compressed into fewer statements, but - * A) the compiler should optimize it - * B) this is not slow - * C) this is an uncommon operation - * D) this is actually readable this way. - */ - lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE; - lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1); - return (pg_tbl)->extended_flag | - (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) | - (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset; + return is_simple ? dev_addr : (pg_tbl->extended_flag | dev_addr); } /* From 35113ccbb530030c8e0dc0e0a2caac3e41cad9c0 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:06:59 -0700 Subject: [PATCH 188/209] staging: gasket: page_table: handle failed dma_map_page Handle dma_map_page failing in gasket_alloc_extended_subtable. Change-Id: I082b0d1a44a9a7c749683cda8daea7e0920d55fc Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor --- gasket_page_table.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gasket_page_table.c b/gasket_page_table.c index 4e84864..4c88b29 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -909,6 +909,22 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, /* Map the page into DMA space. */ pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(pg_tbl->device, pte->dma_addr)) { + dev_dbg(pg_tbl->device, + "%s -> fail to map page %llx " + "[pfn %p phys %p]\n", + __func__, + (unsigned long long)pte->dma_addr, + (void *)page_to_pfn(pte->page), + (void *)page_to_phys(pte->page)); + + /* clean up */ + free_page(page_addr); + vfree(pte->sublevel); + memset(pte, 0, sizeof(struct gasket_page_table_entry)); + + return -ENOMEM; + } /* make the addresses available to the device */ dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; From 2a20ff11720c231130bce2b6c89a1f82f650a5a4 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:10:39 -0700 Subject: [PATCH 189/209] staging: gasket: interrupt: allow device driver to manage interrupts Add interrupt type DEVICE_MANAGED, indicating that the device driver manages interrupt setup and handling. Future non-PCI wire interrupts will use this type, calling gasket_handle_interrupt(), which is now made non-static, to call into the gasket framework in order to update sysfs files for interrupt counts and other framework interfaces. Change-Id: Ie0be3d950ed2706f7ada848c19ddf7017e9623b2 Signed-off-by: Todd Poynor --- gasket_core.h | 1 + gasket_interrupt.c | 18 ++++++++++++++++-- gasket_interrupt.h | 5 +++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/gasket_core.h b/gasket_core.h index 56805d3..ef7cf16 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -56,6 +56,7 @@ enum gasket_interrupt_packing { /* Type of the interrupt supported by the device. */ enum gasket_interrupt_type { PCI_MSIX = 0, + DEVICE_MANAGED = 1, /* Managed externally in device driver */ }; /* diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 9d0ad2c..915146c 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -93,6 +93,9 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) dev_dbg(gasket_dev->dev, "Running interrupt setup\n"); + if (interrupt_data->type == DEVICE_MANAGED) + return; /* device driver handles setup */ + /* Setup the MSIX table. */ for (i = 0; i < interrupt_data->num_interrupts; i++) { @@ -145,7 +148,7 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } } -static void +void gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data, int interrupt_index) { @@ -340,7 +343,6 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev) interrupt_data->interrupts = driver_desc->interrupts; interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index; interrupt_data->pack_width = driver_desc->interrupt_pack_width; - interrupt_data->num_configured = 0; interrupt_data->eventfd_ctxs = kcalloc(driver_desc->num_interrupts, sizeof(struct eventfd_ctx *), @@ -369,6 +371,11 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev) force_msix_interrupt_unmasking(gasket_dev); break; + case DEVICE_MANAGED: /* Device driver manages IRQ init */ + interrupt_data->num_configured = interrupt_data->num_interrupts; + ret = 0; + break; + default: ret = -EINVAL; } @@ -426,6 +433,10 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) force_msix_interrupt_unmasking(gasket_dev); break; + case DEVICE_MANAGED: /* Device driver manages IRQ reinit */ + ret = 0; + break; + default: ret = -EINVAL; } @@ -471,6 +482,9 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) gasket_interrupt_msix_cleanup(interrupt_data); break; + case DEVICE_MANAGED: /* Device driver manages IRQ cleanup */ + break; + default: break; } diff --git a/gasket_interrupt.h b/gasket_interrupt.h index 85526a1..b17b723 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -45,6 +45,11 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev); */ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev); +/* Handle gasket interrupt processing, called from an external handler. */ +void +gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data, + int interrupt_index); + /* * Reset the counts stored in the interrupt subsystem. * @gasket_dev: The Gasket information structure for this device. From 35f51b913e578145b59a17b64c2743205c53d11d Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:14:35 -0700 Subject: [PATCH 190/209] staging: gasket: core: add platform device add and remove functions Add support for platform devices in the gasket framework. Change-Id: I1e3625f9222b37925a2a672c9ea6eed31aefa35f Signed-off-by: Todd Poynor --- gasket_core.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ gasket_core.h | 13 +++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/gasket_core.c b/gasket_core.c index 1cbae4d..65b3ab2 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -668,6 +669,25 @@ lookup_pci_internal_desc(struct pci_dev *pci_dev) return NULL; } +/* + * Registered driver descriptor lookup for platform devices. + * Caller must hold g_mutex. + */ +static struct gasket_internal_desc * +lookup_platform_internal_desc(struct platform_device *pdev) +{ + int i; + + __must_hold(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc && + strcmp(g_descs[i].driver_desc->name, pdev->name) == 0) + return &g_descs[i]; + } + + return NULL; +} + /* * Verifies that the user has permissions to perform the requested mapping and * that the provided descriptor/range is of adequate size to hold the range to @@ -1557,6 +1577,72 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) } EXPORT_SYMBOL(gasket_pci_remove_device); +/* Add platform gasket device. Called by Gasket device probe function. */ +int gasket_platform_add_device(struct platform_device *pdev, + struct gasket_dev **gasket_devp) +{ + int ret; + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev; + struct device *parent; + + dev_dbg(&pdev->dev, "add platform gasket device\n"); + + mutex_lock(&g_mutex); + internal_desc = lookup_platform_internal_desc(pdev); + mutex_unlock(&g_mutex); + if (!internal_desc) { + dev_err(&pdev->dev, + "%s called for unknown driver type\n", __func__); + return -ENODEV; + } + + parent = &pdev->dev; + ret = __gasket_add_device(parent, internal_desc, &gasket_dev); + if (ret) + return ret; + + gasket_dev->platform_dev = pdev; + *gasket_devp = gasket_dev; + return 0; +} +EXPORT_SYMBOL(gasket_platform_add_device); + +/* Remove a platform gasket device. */ +void gasket_platform_remove_device(struct platform_device *pdev) +{ + int i; + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev = NULL; + + /* Find the device desc. */ + mutex_lock(&g_mutex); + internal_desc = lookup_platform_internal_desc(pdev); + mutex_unlock(&g_mutex); + if (!internal_desc) + return; + + /* Now find the specific device */ + mutex_lock(&internal_desc->mutex); + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (internal_desc->devs[i] && + internal_desc->devs[i]->platform_dev == pdev) { + gasket_dev = internal_desc->devs[i]; + break; + } + } + mutex_unlock(&internal_desc->mutex); + + if (!gasket_dev) + return; + + dev_dbg(gasket_dev->dev, "remove %s platform gasket device\n", + internal_desc->driver_desc->name); + + __gasket_remove_device(internal_desc, gasket_dev); +} +EXPORT_SYMBOL(gasket_platform_remove_device); + /** * Lookup a name by number in a num_name table. * @num: Number to lookup. diff --git a/gasket_core.h b/gasket_core.h index ef7cf16..75ad11b 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -265,9 +266,12 @@ struct gasket_dev { /* Device info */ struct device *dev; - /* PCI subsystem metadata. */ + /* PCI device pointer for PCI devices */ struct pci_dev *pci_dev; + /* Platform device pointer for platform devices */ + struct platform_device *platform_dev; + /* This device's index into internal_desc->devs. */ int dev_idx; @@ -539,6 +543,13 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, /* Remove a PCI gasket device. */ void gasket_pci_remove_device(struct pci_dev *pci_dev); +/* Add a platform gasket device. */ +int gasket_platform_add_device(struct platform_device *pdev, + struct gasket_dev **gasket_devp); + +/* Remove a platform gasket device. */ +void gasket_platform_remove_device(struct platform_device *pdev); + /* Enable a Gasket device. */ int gasket_enable_device(struct gasket_dev *gasket_dev); From 4dca66172733d84a0d07bc04dfa7a35c854e8f93 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:17:28 -0700 Subject: [PATCH 191/209] staging: gasket: Kconfig: describe Apex as an Edge TPU device Add a brief description and URL for more information on the Apex device, an Edge TPU (Tensorflow Processing Unit) machine learning accelerator. Change-Id: Ia7b40414e9f0745feb8ced2cb3aa7e9c6baa0c3d Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- Kconfig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Kconfig b/Kconfig index 970e299..e82b855 100644 --- a/Kconfig +++ b/Kconfig @@ -14,8 +14,9 @@ config STAGING_APEX_DRIVER tristate "Apex Driver" depends on STAGING_GASKET_FRAMEWORK help - This driver supports the Apex device. Say Y if you want to - include this driver in the kernel. + This driver supports the Apex Edge TPU device. See + https://cloud.google.com/edge-tpu/ for more information. + Say Y if you want to include this driver in the kernel. To compile this driver as a module, choose M here. The module will be called "apex". From 0dc5779dd7225d4b43b9f918dc7d147609103b77 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:20:41 -0700 Subject: [PATCH 192/209] staging: gasket: Update device virtual address comment Add that number of page table entries and extended address bit offset are configurable. Update example virtual address format to be more consistent with typical usage. Change-Id: I73f9674df74b6d8ee5ed6e5d95ab12dcc56e29cc Signed-off-by: Nick Ewalt Signed-off-by: Todd Poynor Signed-off-by: Greg Kroah-Hartman --- gasket_page_table.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 4c88b29..85a40bb 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -10,10 +10,18 @@ * * This file assumes 4kB pages throughout; can be factored out when necessary. * - * Address format is as follows: + * There is a configurable number of page table entries, as well as a + * configurable bit index for the extended address flag. Both of these are + * specified in gasket_page_table_init through the page_table_config parameter. + * + * The following example assumes: + * page_table_config->total_entries = 8192 + * page_table_config->extended_bit = 63 + * + * Address format: * Simple addresses - those whose containing pages are directly placed in the * device's address translation registers - are laid out as: - * [ 63 - 40: Unused | 39 - 28: 0 | 27 - 12: page index | 11 - 0: page offset ] + * [ 63 - 25: 0 | 24 - 12: page index | 11 - 0: page offset ] * page index: The index of the containing page in the device's address * translation registers. * page offset: The index of the address into the containing page. @@ -21,7 +29,7 @@ * Extended address - those whose containing pages are contained in a second- * level page table whose address is present in the device's address translation * registers - are laid out as: - * [ 63 - 40: Unused | 39: flag | 38 - 37: 0 | 36 - 21: dev/level 0 index | + * [ 63: flag | 62 - 34: 0 | 33 - 21: dev/level 0 index | * 20 - 12: host/level 1 index | 11 - 0: page offset ] * flag: Marker indicating that this is an extended address. Always 1. * dev index: The index of the first-level page in the device's extended From 975c81ef1d5cdcd8200ad30085169c7f14df9add Mon Sep 17 00:00:00 2001 From: Nick Ewalt Date: Tue, 2 Oct 2018 12:27:29 -0700 Subject: [PATCH 193/209] staging: gasket: cleanup extended page table map/unmap Make sure to use dma_sync_single_for_device whenever updating the contents of an extended page table. This performs the cache flush using the correct function from the DMA-API. Also move this call outside of the inner loop to improve performance. Tested: Ran run_tests with inception, testfullyconnected with parameter caching, and rnntransducerdecoderpie107m on AT Enterprise. Change-Id: Ie3c63113dde8cafe448d515e8970810afc9564ac Signed-off-by: Nick Ewalt --- gasket_page_table.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 85a40bb..a617ed0 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -557,17 +557,10 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, /* Make the DMA-space address available to the device. */ dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG; - if (is_simple_mapping) { + if (is_simple_mapping) writeq(dma_addr, &slots[i]); - } else { + else ((u64 __force *)slots)[i] = dma_addr; - /* Extended page table vectors are in DRAM, - * and so need to be synced each time they are updated. - */ - dma_map_single(pg_tbl->device, - (void *)&((u64 __force *)slots)[i], - sizeof(u64), DMA_TO_DEVICE); - } /* Set PTE flags equal to flags param with STATUS=PTE_INUSE. */ ptes[i].flags = SET(FLAGS_STATUS, flags, PTE_INUSE); @@ -639,14 +632,10 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, */ for (i = 0; i < num_pages; i++) { /* release the address from the device, */ - if (is_simple_mapping || - GET(FLAGS_STATUS, ptes[i].flags) == PTE_INUSE) { + if (is_simple_mapping) writeq(0, &slots[i]); - } else { + else ((u64 __force *)slots)[i] = 0; - /* sync above PTE update before updating mappings */ - wmb(); - } /* release the address from the driver, */ if (GET(FLAGS_STATUS, ptes[i].flags) == PTE_INUSE) { @@ -702,6 +691,13 @@ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, gasket_perform_unmapping(pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, len, 0); + /* + * Extended page tables are in DRAM so they need to be + * synced each time they are updated. + */ + dma_sync_single_for_device(pg_tbl->device, + pte->dma_addr + slot_idx * sizeof(u64), + len * sizeof(u64), DMA_TO_DEVICE); } remain -= len; @@ -1045,6 +1041,14 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, return ret; } + /* + * Extended page tables are in DRAM so they need to be synced + * each time they are updated. + */ + dma_sync_single_for_device(pg_tbl->device, + pte->dma_addr + slot_idx * sizeof(u64), + len * sizeof(u64), DMA_TO_DEVICE); + remain -= len; slot_idx = 0; pte++; From e2505d92c7677ff188fec9c6d26f92d53668d42d Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:25:08 -0700 Subject: [PATCH 194/209] char: oscar: gasket: add interface to set a separate DMA device In at least the case of an mfd_cell multi-function device child, the child platform device does not inherit the DMA / IOMMU configuration of the parent. Copying that configuration is non-straightforward or even likely impossible when the parent device is PCI and holds pointers to IOMMU-related state only made available for PCI devices. Add a gasket call to allow platform chip drivers to register their PCI parent as the device to use for DMA API calls, such that the chip drivers can workaround this. Change-Id: I4a4b6de67f5d10197f8ea02d06522090b150d659 Signed-off-by: Todd Poynor --- gasket_core.c | 10 ++++++++++ gasket_core.h | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/gasket_core.c b/gasket_core.c index 65b3ab2..ae3f9c7 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -215,6 +215,7 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, gasket_dev->dev_idx = dev_idx; snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", parent_name); gasket_dev->dev = get_device(parent); + gasket_dev->dma_dev = get_device(parent); /* gasket_bar_data is uninitialized. */ gasket_dev->num_page_tables = driver_desc->num_page_tables; /* max_page_table_size and *page table are uninit'ed */ @@ -247,6 +248,7 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) internal_desc->devs[gasket_dev->dev_idx] = NULL; mutex_unlock(&internal_desc->mutex); put_device(gasket_dev->dev); + put_device(gasket_dev->dma_dev); kfree(gasket_dev); } @@ -1643,6 +1645,14 @@ void gasket_platform_remove_device(struct platform_device *pdev) } EXPORT_SYMBOL(gasket_platform_remove_device); +void gasket_set_dma_device(struct gasket_dev *gasket_dev, + struct device *dma_dev) +{ + put_device(gasket_dev->dma_dev); + gasket_dev->dma_dev = get_device(dma_dev); +} +EXPORT_SYMBOL(gasket_set_dma_device); + /** * Lookup a name by number in a num_name table. * @num: Number to lookup. diff --git a/gasket_core.h b/gasket_core.h index 75ad11b..7301166 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -266,6 +266,9 @@ struct gasket_dev { /* Device info */ struct device *dev; + /* DMA device to use, may be same as above or a parent */ + struct device *dma_dev; + /* PCI device pointer for PCI devices */ struct pci_dev *pci_dev; @@ -550,6 +553,10 @@ int gasket_platform_add_device(struct platform_device *pdev, /* Remove a platform gasket device. */ void gasket_platform_remove_device(struct platform_device *pdev); +/* Set DMA device to use (if different from PCI/platform device) */ +void gasket_set_dma_device(struct gasket_dev *gasket_dev, + struct device *dma_dev); + /* Enable a Gasket device. */ int gasket_enable_device(struct gasket_dev *gasket_dev); From be43ad6a89125541e24eeb6151298dba0edabf6f Mon Sep 17 00:00:00 2001 From: Waqar Hameed Date: Mon, 15 Jul 2019 19:22:58 -0700 Subject: [PATCH 195/209] staging: gasket: Fix dev_addr truncation on 32-bit arch The extended address bit is at position 63 (c.f. apex_driver.c:60). The device address will thus not fit in a ulong on 32-bit ARM systems. Use u64 instead. [toddpoynor@google.com: more fixups, fix debug printk param order] Bug: 137378503 Signed-off-by: Waqar Hameed Change-Id: If801b2b97d88c1329f3a359507e265837150571c --- gasket_interrupt.c | 4 ++-- gasket_page_table.c | 29 +++++++++++++++-------------- gasket_page_table.h | 4 ++-- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 915146c..1e8b960 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -81,8 +81,8 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) { int i; int pack_shift; - ulong mask; - ulong value; + u64 mask; + u64 value; struct gasket_interrupt_data *interrupt_data = gasket_dev->interrupt_data; diff --git a/gasket_page_table.c b/gasket_page_table.c index a617ed0..b5438da 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -725,11 +725,11 @@ static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, * Extended page 1000, offset 511: * Input (0, 1000, 511), Output 0x8003E81FF */ -static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, +static u64 gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, int is_simple, uint page_index, uint offset) { - ulong dev_addr = (page_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; + u64 dev_addr = (page_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; return is_simple ? dev_addr : (pg_tbl->extended_flag | dev_addr); } @@ -791,8 +791,8 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, /* check if the device address is out of bound */ addr = dev_addr & ~((pg_tbl)->extended_flag); if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - dev_err(pg_tbl->device, "device address out of bounds: 0x%llx\n", - dev_addr); + dev_err(pg_tbl->device, + "device address out of bounds: 0x%llx\n", dev_addr); return true; } @@ -853,7 +853,7 @@ static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, * If there is an error, no pages are mapped. */ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, + ulong host_addr, u64 dev_addr, uint num_pages, u32 flags) { int ret; @@ -863,7 +863,8 @@ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, if (ret) { dev_err(pg_tbl->device, "page table slots %u (@ 0x%llx) to %u are not available\n", - slot_idx, dev_addr, slot_idx + num_pages - 1); + slot_idx, (long long unsigned int)dev_addr, + slot_idx + num_pages - 1); return ret; } @@ -999,11 +1000,11 @@ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, * If there is an error, no pages are mapped. */ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, + ulong host_addr, u64 dev_addr, uint num_pages, u32 flags) { int ret; - ulong dev_addr_end; + u64 dev_addr_end; uint slot_idx, remain, len; struct gasket_page_table_entry *pte; u64 __iomem *slot_base; @@ -1012,11 +1013,11 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, if (ret) { dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; dev_err(pg_tbl->device, - "page table slots (%lu,%llu) (@ 0x%lx) to (%lu,%lu) are " - "not available\n", + "page table slots (%lu,%lu) (@ 0x%llx) to (%lu,%lu) " + "are not available\n", gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), - dev_addr, gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), + (long long unsigned int)dev_addr, gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end), gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end)); return ret; @@ -1067,7 +1068,7 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, * The page table mutex is held for the entire operation. */ int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, - ulong dev_addr, uint num_pages, u32 flags) + u64 dev_addr, uint num_pages, u32 flags) { int ret; @@ -1194,7 +1195,7 @@ fail: /* See gasket_page_table.h for description. */ bool gasket_page_table_are_addrs_bad( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + struct gasket_page_table *pg_tbl, ulong host_addr, u64 dev_addr, ulong bytes) { if (host_addr & (PAGE_SIZE - 1)) { @@ -1352,7 +1353,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, gasket_dev->page_table[index]->coherent_pages[j].paddr = handle + j * PAGE_SIZE; gasket_dev->page_table[index]->coherent_pages[j].kernel_virt = - (dma_addr_t)mem + j * PAGE_SIZE; + (ulong)mem + j * PAGE_SIZE; } return 0; diff --git a/gasket_page_table.h b/gasket_page_table.h index e61b07a..609e1d9 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -97,7 +97,7 @@ int gasket_page_table_partition(struct gasket_page_table *page_table, * If there is an error, no pages are mapped. */ int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, - ulong dev_addr, uint num_pages, u32 flags); + u64 dev_addr, uint num_pages, u32 flags); /* * Un-map host pages from device memory. @@ -165,7 +165,7 @@ int gasket_page_table_lookup_page(struct gasket_page_table *page_table, * Returns true if the mapping is bad, false otherwise. */ bool gasket_page_table_are_addrs_bad(struct gasket_page_table *page_table, - ulong host_addr, ulong dev_addr, + ulong host_addr, u64 dev_addr, ulong bytes); /* From 4a701a4c3181135ba87b961bec5b15090552cf73 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Tue, 27 Aug 2019 00:51:22 -0700 Subject: [PATCH 196/209] staging: gasket: fix mmap of coherent buffer when IOMMU translation on Coherent buffer mmap code assumes DMA addresses are physical addresses, which is patently wrong when a downstream IOMMU allocates and translates IOVAs for DMA mappings. Use dma_mmap_coherent(). Bug: 140076240 Bug: 111709861 Change-Id: Ieaa052378844aaf126ea3bc9dba544e7edb37bc7 Signed-off-by: Todd Poynor --- gasket_core.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index ae3f9c7..33a7372 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -1009,13 +1009,14 @@ static int gasket_mmap_coherent(struct gasket_dev *gasket_dev, } vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - ret = remap_pfn_range(vma, vma->vm_start, - (gasket_dev->coherent_buffer.phys_base) >> - PAGE_SHIFT, requested_length, vma->vm_page_prot); + vma->vm_pgoff = 0; + ret = dma_mmap_coherent(gasket_dev->dma_dev, vma, + gasket_dev->coherent_buffer.virt_base, + gasket_dev->coherent_buffer.phys_base, + requested_length); if (ret) { - dev_err(gasket_dev->dev, "Error remapping PFN range err=%d.\n", - ret); + dev_err(gasket_dev->dev, + "Error mmapping coherent buffer err=%d.\n", ret); trace_gasket_mmap_exit(ret); return ret; } From 067d10799a0a67c8fd5320a40ac7d55432f52638 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:46:23 -0700 Subject: [PATCH 197/209] staging: gasket: remove unnecessary defines Change-Id: I3c2a2c435671b6a41d9294d444e9200aae13b30b Signed-off-by: Leonid Lobachev --- gasket_core.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gasket_core.h b/gasket_core.h index 7301166..7405b6a 100644 --- a/gasket_core.h +++ b/gasket_core.h @@ -18,12 +18,6 @@ #include #include -#define writeq_relaxed writeq -#define writeq(__value, __reg) \ - (*(volatile u64 __force *)(__reg) = (cpu_to_le64(__value))) -#define readq_relaxed readq -#define readq(__reg) le64_to_cpu(*(volatile u64 __force *)(__reg)) - #include "gasket_constants.h" /** From ee042622739f91e9ac831cc31b4f3fff5bbd464d Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Mon, 23 Sep 2019 14:54:37 -0700 Subject: [PATCH 198/209] staging: gasket: fix ISO C90 warnings Change-Id: I0e114cf27e1f38830c04bdd0f5f705b9a795fe59 Signed-off-by: Leonid Lobachev --- apex_driver.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 7389cf8..5a2b45a 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -899,7 +899,7 @@ static void apply_module_params(struct apex_dev *apex_dev) { } static void check_temperature_work_handler(struct work_struct *work) { - int i; + int i, temp_poll_interval; u32 adc_temp, clk_div, tmp; const u32 mask = ((1 << 2) - 1) << 28; struct apex_dev *apex_dev = @@ -937,7 +937,7 @@ static void check_temperature_work_handler(struct work_struct *work) { mutex_unlock(&gasket_dev->mutex); - int temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); + temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); if (temp_poll_interval > 0) schedule_delayed_work(&apex_dev->check_temperature_work, msecs_to_jiffies(temp_poll_interval)); @@ -963,7 +963,7 @@ DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, static int apex_pci_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) { - int ret; + int ret, temp_poll_interval; ulong page_table_ready, msix_table_ready; int retries = 0; struct gasket_dev *gasket_dev; @@ -1057,7 +1057,7 @@ static int apex_pci_probe(struct pci_dev *pci_dev, apex_enter_reset(gasket_dev); /* Enable thermal polling */ - int temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); + temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); if (temp_poll_interval > 0) schedule_delayed_work(&apex_dev->check_temperature_work, msecs_to_jiffies(temp_poll_interval)); From f003a969092fdf55823ee2a255985d6cf1872e09 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Wed, 25 Sep 2019 10:43:59 -0700 Subject: [PATCH 199/209] staging: gasket: add dma_bit_mask module parameter. This allows to force dma buffers to be located in 32bit low address space to workaround issues with some pcie controllers. Change-Id: I7f77081882b6dffb91f034f8491ad8ffa1375fe7 Signed-off-by: Leonid Lobachev --- gasket_core.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 33a7372..16112bb 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -103,6 +103,16 @@ enum gasket_sysfs_attribute_type { ATTR_USER_MEM_RANGES }; +/* On some arm64 systems pcie dma controller can only access lower 4GB of + * addresses. Unfortunately vendor BSP isn't providing any means of determining + * this limitation and there're no errors reported if access to higher addresses + * if being done. This parameter allows to workaround this issue by pretending + * that our device only supports 32 bit addresses. This in turn will cause + * dma driver to use shadow buffers located in low 32 bit address space. + */ +static int dma_bit_mask = 64; +module_param(dma_bit_mask, int, 0644); + /* Perform a standard Gasket callback. */ static inline int check_and_invoke_callback(struct gasket_dev *gasket_dev, @@ -316,8 +326,9 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) goto fail; } - dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64)); - dma_set_coherent_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64)); + dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(dma_bit_mask)); + dma_set_coherent_mask(&gasket_dev->pci_dev->dev, + DMA_BIT_MASK(dma_bit_mask)); return 0; From a9b97c3f9235a9ab4dec813daba3211e73a05fec Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Fri, 18 Oct 2019 11:00:24 -0700 Subject: [PATCH 200/209] staging: gasket: add unique chip id sysfs node Change-Id: I15ee278e212b76dfd649f03f256870b9872c3395 Signed-off-by: Leonid Lobachev --- apex_driver.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/apex_driver.c b/apex_driver.c index 5a2b45a..d7b9ac9 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -89,6 +89,7 @@ enum sysfs_attribute_type { ATTR_TEMP_TRIP1, ATTR_TEMP_TRIP2, ATTR_TEMP_POLL_INTERVAL, + ATTR_UNIQUE_ID, }; /* @@ -124,6 +125,10 @@ enum apex_bar2_regs { APEX_BAR2_REG_OMC0_D4 = 0x01a0d4, APEX_BAR2_REG_OMC0_D8 = 0x01a0d8, APEX_BAR2_REG_OMC0_DC = 0x01a0dc, + APEX_BAR2_REG_EFUSE_DC = 0x01a2dc, + APEX_BAR2_REG_EFUSE_E0 = 0x01a2e0, + APEX_BAR2_REG_EFUSE_E4 = 0x01a2e4, + APEX_BAR2_REG_EFUSE_E8 = 0x01a2e8, /* Error registers - Used mostly for debug */ APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0, @@ -635,7 +640,7 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, char *buf) { int ret; - unsigned value; + unsigned value, value2, value3, value4; struct gasket_dev *gasket_dev; struct apex_dev *apex_dev; struct gasket_sysfs_attribute *gasket_attr; @@ -722,6 +727,19 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, ret = scnprintf(buf, PAGE_SIZE, "%i\n", atomic_read(&apex_dev->temp_poll_interval)); break; + case ATTR_UNIQUE_ID: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_DC); + value2 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_E0); + value3 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_E4); + value4 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_E8); + ret = snprintf(buf, PAGE_SIZE, "%.8x%.8x%.8x%.8x\n", value4, + value3, value2, value); + break; + default: dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", attr->attr.name); @@ -857,6 +875,7 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { ATTR_TEMP_TRIP2), GASKET_SYSFS_RW(temp_poll_interval, sysfs_show, sysfs_store, ATTR_TEMP_POLL_INTERVAL), + GASKET_SYSFS_RO(unique_id, sysfs_show, ATTR_UNIQUE_ID), GASKET_END_OF_ATTR_ARRAY }; From 6535f9ada181dd8ac72e8c0b09d9121d2f55e221 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Wed, 13 Nov 2019 17:54:35 -0800 Subject: [PATCH 201/209] Update gasket framework version. This is needed to avoid the need for manual dkms driver installation with --force on systems where upstream staging kernel module is present already. Change-Id: Ifb3dc930757835888ae3e13d1607d5fd7bc710ed --- gasket_constants.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_constants.h b/gasket_constants.h index 50d87c7..ff113bc 100644 --- a/gasket_constants.h +++ b/gasket_constants.h @@ -3,7 +3,7 @@ #ifndef __GASKET_CONSTANTS_H__ #define __GASKET_CONSTANTS_H__ -#define GASKET_FRAMEWORK_VERSION "1.1.2" +#define GASKET_FRAMEWORK_VERSION "1.1.3" /* * The maximum number of simultaneous device types supported by the framework. From 0d361c049302e06e6df506499882efb8740de0da Mon Sep 17 00:00:00 2001 From: Jonas Larsson Date: Tue, 18 Feb 2020 13:14:50 -0800 Subject: [PATCH 202/209] staging: gasket: support mapping of dma-bufs. This allows userspace to pass in dma-buf file descriptors for mapping in addition to userspace virtual memory pointers, leaving cache maintenance to the dma-buf exporter and userspace application. Bug: 149764192 Change-Id: Iee8b581db0a461fe13ee8712d47d66f8fb16e7c9 Signed-off-by: Jonas Larsson --- gasket.h | 22 ++++++ gasket_ioctl.c | 37 +++++++++ gasket_page_table.c | 188 +++++++++++++++++++++++++++++++++++++++++--- gasket_page_table.h | 32 ++++++++ 4 files changed, 270 insertions(+), 9 deletions(-) diff --git a/gasket.h b/gasket.h index 93e7af1..bdf468f 100644 --- a/gasket.h +++ b/gasket.h @@ -74,6 +74,21 @@ struct gasket_coherent_alloc_config_ioctl { u64 dma_address; }; +/* + * Common structure for ioctls mapping and unmapping dma-bufs when using the + * Gasket page_table module. + * map: boolean, non-zero to map, 0 to unmap. + * flags: see gasket_page_table_ioctl_flags.flags. + */ +struct gasket_page_table_ioctl_dmabuf { + u64 page_table_index; + u64 device_address; + int dmabuf_fd; + u32 num_pages; + u32 map; + u32 flags; +}; + /* Base number for all Gasket-common IOCTLs */ #define GASKET_IOCTL_BASE 0xDC @@ -152,4 +167,11 @@ struct gasket_coherent_alloc_config_ioctl { #define GASKET_IOCTL_MAP_BUFFER_FLAGS \ _IOW(GASKET_IOCTL_BASE, 12, struct gasket_page_table_ioctl_flags) +/* + * Tells the kernel to map/unmap dma-buf with fd to device_address in + * page_table_index page table. + */ +#define GASKET_IOCTL_MAP_DMABUF \ + _IOW(GASKET_IOCTL_BASE, 13, struct gasket_page_table_ioctl_dmabuf) + #endif /* __GASKET_H__ */ diff --git a/gasket_ioctl.c b/gasket_ioctl.c index 4696007..1760a9c 100644 --- a/gasket_ioctl.c +++ b/gasket_ioctl.c @@ -212,6 +212,38 @@ static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, return 0; } +/* Map/unmap dma-buf to/from a device virtual address. */ +static int gasket_map_dmabuf(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl_dmabuf __user *argp) +{ + struct gasket_page_table_ioctl_dmabuf dbuf; + struct gasket_page_table *pg_tbl; + + if (copy_from_user(&dbuf, argp, sizeof(dbuf))) + return -EFAULT; + + if (dbuf.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + pg_tbl = gasket_dev->page_table[dbuf.page_table_index]; + if (gasket_page_table_is_dev_addr_bad(pg_tbl, + dbuf.device_address, + dbuf.num_pages * PAGE_SIZE)) + return -EINVAL; + + if (dbuf.map) + return gasket_page_table_map_dmabuf(pg_tbl, + dbuf.dmabuf_fd, + dbuf.device_address, + dbuf.num_pages, + dbuf.flags); + else + return gasket_page_table_unmap_dmabuf(pg_tbl, + dbuf.dmabuf_fd, + dbuf.device_address, + dbuf.num_pages); +} + /* * Reserve structures for coherent allocation, and allocate or free the * corresponding memory. @@ -291,6 +323,7 @@ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) case GASKET_IOCTL_MAP_BUFFER: case GASKET_IOCTL_MAP_BUFFER_FLAGS: case GASKET_IOCTL_UNMAP_BUFFER: + case GASKET_IOCTL_MAP_DMABUF: return alive && write; case GASKET_IOCTL_CLEAR_EVENTFD: @@ -388,6 +421,9 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) trace_gasket_ioctl_integer_data(0); retval = gasket_interrupt_reset_counts(gasket_dev); break; + case GASKET_IOCTL_MAP_DMABUF: + retval = gasket_map_dmabuf(gasket_dev, argp); + break; default: /* If we don't understand the ioctl, the best we can do is trace * the arg. @@ -424,6 +460,7 @@ long gasket_is_supported_ioctl(uint cmd) case GASKET_IOCTL_MAP_BUFFER: case GASKET_IOCTL_MAP_BUFFER_FLAGS: case GASKET_IOCTL_UNMAP_BUFFER: + case GASKET_IOCTL_MAP_DMABUF: case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: return 1; diff --git a/gasket_page_table.c b/gasket_page_table.c index b5438da..5ae2024 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -42,9 +42,11 @@ #include "gasket_page_table.h" #include +#include #include #include #include +#include #include #include #include @@ -177,6 +179,15 @@ struct gasket_coherent_page_entry { u32 in_use; }; +/* Storage for dmabuf mapping information. */ +struct gasket_dmabuf_mapping { + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; + enum dma_data_direction direction; + struct list_head list; +}; + /* * [Host-side] page table descriptor. * @@ -237,6 +248,9 @@ struct gasket_page_table { * gasket_mmap function, so user_virt belongs in the driver anyhow. */ struct gasket_coherent_page_entry *coherent_pages; + + /* List of dmabufs currently attached and mapped. */ + struct list_head dmabufs; }; /* See gasket_page_table.h for description. */ @@ -306,6 +320,7 @@ int gasket_page_table_init(struct gasket_page_table **ppg_tbl, (u64 __iomem *)&bar_data->virt_base[page_table_config->extended_reg]; pg_tbl->device = get_device(device); pg_tbl->pci_dev = pci_dev; + INIT_LIST_HEAD(&pg_tbl->dmabufs); dev_dbg(device, "Page table initialized successfully\n"); @@ -481,7 +496,9 @@ static bool gasket_release_page(struct page *page) */ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, - u64 __iomem *slots, ulong host_addr, + u64 __iomem *slots, + struct sg_page_iter *sg_iter, + ulong host_addr, uint num_pages, u32 flags, int is_simple_mapping) { @@ -492,6 +509,12 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ulong page_addr; int i; + /* Must have a virtual host address or a sg iterator, but not both. */ + if(!((uintptr_t)host_addr ^ (uintptr_t)sg_iter)) { + dev_err(pg_tbl->device, "need sg_iter or host_addr\n"); + return -EINVAL; + } + if (GET(FLAGS_DMA_DIRECTION, flags) == DMA_NONE) { dev_err(pg_tbl->device, "invalid DMA direction flags=0x%lx\n", (unsigned long)flags); @@ -502,7 +525,15 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, page_addr = host_addr + i * PAGE_SIZE; offset = page_addr & (PAGE_SIZE - 1); dev_dbg(pg_tbl->device, "%s i %d\n", __func__, i); - if (is_coherent(pg_tbl, host_addr)) { + if (sg_iter) { + if (!__sg_page_iter_next(sg_iter)) + return -EINVAL; + + /* Page already mapped for DMA. */ + ptes[i].dma_addr = sg_page_iter_dma_address(sg_iter); + ptes[i].page = NULL; + offset = 0; + } else if (is_coherent(pg_tbl, host_addr)) { u64 off = (u64)host_addr - (u64)pg_tbl->coherent_pages[0].user_virt; @@ -853,6 +884,7 @@ static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, * If there is an error, no pages are mapped. */ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, + struct sg_page_iter *sg_iter, ulong host_addr, u64 dev_addr, uint num_pages, u32 flags) { @@ -869,8 +901,8 @@ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, } ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx, - pg_tbl->base_slot + slot_idx, host_addr, - num_pages, flags, 1); + pg_tbl->base_slot + slot_idx, sg_iter, + host_addr, num_pages, flags, 1); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); @@ -1000,6 +1032,7 @@ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, * If there is an error, no pages are mapped. */ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, + struct sg_page_iter *sg_iter, ulong host_addr, u64 dev_addr, uint num_pages, u32 flags) { @@ -1034,8 +1067,8 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, slot_base = (u64 __iomem *)(page_address(pte->page) + pte->offset); ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx, - slot_base + slot_idx, host_addr, - len, flags, 0); + slot_base + slot_idx, sg_iter, + host_addr, len, flags, 0); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); @@ -1053,7 +1086,8 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, remain -= len; slot_idx = 0; pte++; - host_addr += len * PAGE_SIZE; + if (host_addr) + host_addr += len * PAGE_SIZE; } return 0; @@ -1078,10 +1112,10 @@ int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, mutex_lock(&pg_tbl->mutex); if (gasket_addr_is_simple(pg_tbl, dev_addr)) { - ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr, + ret = gasket_map_simple_pages(pg_tbl, NULL, host_addr, dev_addr, num_pages, flags); } else { - ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr, + ret = gasket_map_extended_pages(pg_tbl, NULL, host_addr, dev_addr, num_pages, flags); } @@ -1116,8 +1150,144 @@ void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, u64 dev_addr, } EXPORT_SYMBOL(gasket_page_table_unmap); +int gasket_page_table_map_dmabuf(struct gasket_page_table *pg_tbl, int fd, + u64 dev_addr, uint num_pages, u32 flags) +{ + int ret, locked = 0; + struct dma_buf *dmabuf = NULL; + struct dma_buf_attachment *attachment = NULL; + struct sg_table *sgt = NULL; + struct sg_page_iter sg_iter; + struct gasket_dmabuf_mapping *mapping = NULL; + enum dma_data_direction direction = GET(FLAGS_DMA_DIRECTION, flags); + + if (direction == DMA_NONE) { + dev_err(pg_tbl->device, + "invalid DMA direction flags=0x%x\n", flags); + return -EINVAL; + } + + if (!num_pages) + return 0; + + dmabuf = dma_buf_get(fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + if (PAGE_ALIGN(dmabuf->size) / PAGE_SIZE < num_pages) + return -EINVAL; + + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) { + ret = -ENOMEM; + goto out; + } + + attachment = dma_buf_attach(dmabuf, pg_tbl->device); + if (IS_ERR(attachment)) { + ret = PTR_ERR(attachment); + goto out; + } + + sgt = dma_buf_map_attachment(attachment, direction); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + goto out; + } + + mutex_lock(&pg_tbl->mutex); + locked = 1; + + __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->nents, 0); + if (gasket_addr_is_simple(pg_tbl, dev_addr)) { + ret = gasket_map_simple_pages(pg_tbl, &sg_iter, 0, dev_addr, + num_pages, flags); + } else { + ret = gasket_map_extended_pages(pg_tbl, &sg_iter, 0, dev_addr, + num_pages, flags); + } + + if (!ret) { + INIT_LIST_HEAD(&mapping->list); + get_dma_buf(dmabuf); + mapping->dmabuf = dmabuf; + mapping->attachment = attachment; + mapping->sgt = sgt; + mapping->direction = direction; + list_add(&mapping->list, &pg_tbl->dmabufs); + sgt = NULL; + attachment = NULL; + mapping = NULL; + } + +out: + if (locked) + mutex_unlock(&pg_tbl->mutex); + + if (!IS_ERR_OR_NULL(sgt)) + dma_buf_unmap_attachment(attachment, sgt, direction); + + if (!IS_ERR_OR_NULL(attachment)) + dma_buf_detach(dmabuf, attachment); + + kfree(mapping); + dma_buf_put(dmabuf); + + return ret; +} +EXPORT_SYMBOL(gasket_page_table_map_dmabuf); + +/* Detach dmabuf from our device if attached, NULL to detach all. */ +static void gasket_page_table_detach_dmabuf_nolock(struct gasket_page_table *pg_tbl, + struct dma_buf *dmabuf) +{ + struct gasket_dmabuf_mapping *mapping, *tmp; + + list_for_each_entry_safe(mapping, tmp, &pg_tbl->dmabufs, list) { + if (!dmabuf || mapping->dmabuf == dmabuf) { + dma_buf_unmap_attachment(mapping->attachment, + mapping->sgt, + mapping->direction); + dma_buf_detach(mapping->dmabuf, mapping->attachment); + dma_buf_put(mapping->dmabuf); + list_del(&mapping->list); + kfree(mapping); + } + } +} + +int gasket_page_table_unmap_dmabuf(struct gasket_page_table *pg_tbl, int fd, + u64 dev_addr, uint num_pages) +{ + struct dma_buf *dmabuf; + struct gasket_dmabuf_mapping *mapping, *tmp; + + dmabuf = dma_buf_get(fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + if (PAGE_ALIGN(dmabuf->size) / PAGE_SIZE < num_pages) { + dma_buf_put(dmabuf); + return -EINVAL; + } + + mutex_lock(&pg_tbl->mutex); + + gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); + gasket_page_table_detach_dmabuf_nolock(pg_tbl, dmabuf); + + mutex_unlock(&pg_tbl->mutex); + + dma_buf_put(dmabuf); + + return 0; +} +EXPORT_SYMBOL(gasket_page_table_unmap_dmabuf); + static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl) { + gasket_page_table_detach_dmabuf_nolock(pg_tbl, NULL); + gasket_unmap_simple_pages(pg_tbl, gasket_components_to_dev_address(pg_tbl, 1, 0, 0), diff --git a/gasket_page_table.h b/gasket_page_table.h index 609e1d9..c203c57 100644 --- a/gasket_page_table.h +++ b/gasket_page_table.h @@ -99,6 +99,38 @@ int gasket_page_table_partition(struct gasket_page_table *page_table, int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, u64 dev_addr, uint num_pages, u32 flags); +/* + * Map dma-buf pages into device memory. + * @page_table: Gasket page table pointer. + * @fd: Dma-buf file descriptor. + * @dev_addr: Starting device address of the pages. + * @num_pages: Number of [4kB] pages to map. + * @flags: Specifies attributes to apply to the pages. + * Internal structure matches gasket_page_table_ioctl_flags.flags. + * + * Description: Maps "num_pages" pages of dma-buf pointed to by + * fd to the address "dev_addr" in device memory. + * + * The caller is responsible for checking the dev_addr range. + * + * Returns 0 if successful or a non-zero error number otherwise. + * If there is an error, no pages are mapped. + */ +int gasket_page_table_map_dmabuf(struct gasket_page_table *page_table, int fd, + u64 dev_addr, uint num_pages, u32 flags); + +/* + * Unmap dma-buf pages from device memory. + * @page_table: Gasket page table pointer. + * @fd: Dma-buf file descriptor. + * @dev_addr: Starting device address of the pages. + * @num_pages: Number of [4kB] pages to map. + * + * Description: The inverse of gasket_page_table_map_dmabuf. + */ +int gasket_page_table_unmap_dmabuf(struct gasket_page_table *page_table, int fd, + u64 dev_addr, uint num_pages); + /* * Un-map host pages from device memory. * @page_table: Gasket page table pointer. From fa575cdad7a6e653f791531c990c1e768571fb2f Mon Sep 17 00:00:00 2001 From: Jonas Larsson Date: Thu, 5 Mar 2020 14:19:38 -0800 Subject: [PATCH 203/209] staging: gasket: don't require all pages to be writable Buffers to be mapped as DMA_TO_DEVICE must be writable due to the 'write' argument to get_user_pages_fast being hard coded to 1. This prevents userspace from passing in pointers returned from mmap(fd, PROT_READ), instead mmap(fd, PROT_READ | PROT_WRITE) must be used even if the buffer is supposed to be read only. Instead set writable to non zero iff direction != DMA_TO_DEVICE, ie only buffers with DMA_FROM_DEVICE or DMA_BIDIRECTIONAL are required to be writable. Change-Id: I21c97b2bb855e11ecaa6ae9e81cf8b463b4aedaa Signed-off-by: Jonas Larsson --- gasket_page_table.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 5ae2024..0740c72 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -508,6 +508,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, dma_addr_t dma_addr; ulong page_addr; int i; + enum dma_data_direction direction; /* Must have a virtual host address or a sg iterator, but not both. */ if(!((uintptr_t)host_addr ^ (uintptr_t)sg_iter)) { @@ -515,7 +516,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, return -EINVAL; } - if (GET(FLAGS_DMA_DIRECTION, flags) == DMA_NONE) { + direction = GET(FLAGS_DMA_DIRECTION, flags); + if (direction == DMA_NONE) { dev_err(pg_tbl->device, "invalid DMA direction flags=0x%lx\n", (unsigned long)flags); return -EINVAL; @@ -542,7 +544,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + off + i * PAGE_SIZE; } else { - ret = get_user_pages_fast(page_addr - offset, 1, 1, + ret = get_user_pages_fast(page_addr - offset, 1, + direction != DMA_TO_DEVICE, &page); if (ret <= 0) { From cf417597710e60d31e6736a75056bf33857ede3e Mon Sep 17 00:00:00 2001 From: Waqar Hameed Date: Mon, 6 Apr 2020 14:02:05 +0200 Subject: [PATCH 204/209] staging: gasket: Fix formatting Add a space before the open parentheses in an if-statement, to follow Linux kernel coding style. Signed-off-by: Waqar Hameed Change-Id: Ic6658aecdde508762117364669872d69dcb0e8e6 --- gasket_page_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index 0740c72..d30247e 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -511,7 +511,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, enum dma_data_direction direction; /* Must have a virtual host address or a sg iterator, but not both. */ - if(!((uintptr_t)host_addr ^ (uintptr_t)sg_iter)) { + if (!((uintptr_t)host_addr ^ (uintptr_t)sg_iter)) { dev_err(pg_tbl->device, "need sg_iter or host_addr\n"); return -EINVAL; } From 3b6740a1d496b85d99765d1ae179c68ced4a13c5 Mon Sep 17 00:00:00 2001 From: Waqar Hameed Date: Mon, 6 Apr 2020 13:40:44 +0200 Subject: [PATCH 205/209] staging: gasket: Remove unused local variables The variables "mapping" and "tmp" are unused in gasket_page_table_unmap_dmabuf(). Signed-off-by: Waqar Hameed Change-Id: I0da67e5e91fb8c8f34b13e7227afb8410bcbc138 --- gasket_page_table.c | 1 - 1 file changed, 1 deletion(-) diff --git a/gasket_page_table.c b/gasket_page_table.c index d30247e..e495658 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -1263,7 +1263,6 @@ int gasket_page_table_unmap_dmabuf(struct gasket_page_table *pg_tbl, int fd, u64 dev_addr, uint num_pages) { struct dma_buf *dmabuf; - struct gasket_dmabuf_mapping *mapping, *tmp; dmabuf = dma_buf_get(fd); if (IS_ERR(dmabuf)) From b20a2f21c3bbe8410e0782f4b0d5173064062b8c Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Thu, 4 Jun 2020 21:25:43 -0700 Subject: [PATCH 206/209] staging: gasket: apex: restore thermal control registers After host suspend/resume operation device is power cycled and loses all custom register settings. All settings are restored on first device open except thermal monitoring related settings. Signed-off-by: Leonid Lobachev Change-Id: I3a7c69efd919da0992c585874d7fe906a7768ee7 --- apex_driver.c | 105 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index d7b9ac9..1d7b7ce 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -73,6 +73,10 @@ struct apex_dev { struct delayed_work check_temperature_work; u32 adc_trip_points[3]; atomic_t temp_poll_interval; + u32 hw_temp_warn1_adc; + u32 hw_temp_warn2_adc; + bool hw_temp_warn1_en; + bool hw_temp_warn2_en; }; /* Enumeration of the supported sysfs entries. */ @@ -690,26 +694,20 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); break; case ATTR_TEMP_WARN1: - value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D4); - value = (value >> 16) & ((1 << 10) - 1); - ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->hw_temp_warn1_adc)); break; case ATTR_TEMP_WARN2: - value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D8); - value = (value >> 16) & ((1 << 10) - 1); - ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->hw_temp_warn2_adc)); break; case ATTR_TEMP_WARN1_EN: - value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D4); - ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + apex_dev->hw_temp_warn1_en); break; case ATTR_TEMP_WARN2_EN: - value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D8); - ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + apex_dev->hw_temp_warn2_en); break; case ATTR_TEMP_TRIP0: ret = scnprintf(buf, PAGE_SIZE, "%i\n", @@ -792,24 +790,28 @@ static ssize_t sysfs_store(struct device *device, struct device_attribute *attr, gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D4, value, 10, 16); + apex_dev->hw_temp_warn1_adc = value; break; case ATTR_TEMP_WARN2: value = millic_to_adc(value); gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D8, value, 10, 16); + apex_dev->hw_temp_warn2_adc = value; break; case ATTR_TEMP_WARN1_EN: value = value > 0 ? 1 : 0; gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D4, value, 1, 31); + apex_dev->hw_temp_warn1_en = !!value; break; case ATTR_TEMP_WARN2_EN: value = value > 0 ? 1 : 0; gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D8, value, 1, 31); + apex_dev->hw_temp_warn2_en = !!value; break; case ATTR_TEMP_TRIP0: value = millic_to_adc(value); @@ -879,6 +881,7 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { GASKET_END_OF_ATTR_ARRAY }; +/* Stores kernel module parameters to device specific data buffer */ static void apply_module_params(struct apex_dev *apex_dev) { kernel_param_lock(THIS_MODULE); @@ -898,23 +901,47 @@ static void apply_module_params(struct apex_dev *apex_dev) { apex_dev->adc_trip_points[2] = millic_to_adc(trip_point2_temp); atomic_set(&apex_dev->temp_poll_interval, temp_poll_interval); + apex_dev->hw_temp_warn1_adc = millic_to_adc(hw_temp_warn1); + apex_dev->hw_temp_warn2_adc = millic_to_adc(hw_temp_warn2); + apex_dev->hw_temp_warn1_en = hw_temp_warn1_en; + apex_dev->hw_temp_warn2_en = hw_temp_warn2_en; + + kernel_param_unlock(THIS_MODULE); +} + +/* Applies hw temp warning settings to device */ +static void program_hw_temp_warnings(struct apex_dev *apex_dev) { gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D4, - millic_to_adc(hw_temp_warn1), 10, 16); + apex_dev->hw_temp_warn1_adc, 10, 16); gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D8, - millic_to_adc(hw_temp_warn2), 10, 16); - if (hw_temp_warn1_en) + apex_dev->hw_temp_warn2_adc, 10, 16); + if (apex_dev->hw_temp_warn1_en) gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D4, 1, 1, 31); - if (hw_temp_warn2_en) + if (apex_dev->hw_temp_warn2_en) gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, APEX_BAR2_REG_OMC0_D8, 1, 1, 31); +} - kernel_param_unlock(THIS_MODULE); +static void enable_thermal_sensing(struct gasket_dev *gasket_dev) { + // Enable thermal sensor clocks + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D0, 0x1, 1, 7); + + // Enable thermal sensor (ENAD ENVR ENBG) + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, 0x7, 3, 0); + + // Enable OMC thermal sensor controller + // This bit should be asserted 100 us after ENAD ENVR ENBG + schedule_timeout(usecs_to_jiffies(100)); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC, 0x1, 1, 0); } static void check_temperature_work_handler(struct work_struct *work) { @@ -1021,6 +1048,7 @@ static int apex_pci_probe(struct pci_dev *pci_dev, check_temperature_work_handler); apex_dev->gasket_dev_ptr = gasket_dev; apply_module_params(apex_dev); + program_hw_temp_warnings(apex_dev); pci_set_drvdata(pci_dev, apex_dev); apex_reset(gasket_dev); @@ -1046,19 +1074,7 @@ static int apex_pci_probe(struct pci_dev *pci_dev, goto remove_device; } - // Enable thermal sensor clocks - gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D0, 0x1, 1, 7); - - // Enable thermal sensor (ENAD ENVR ENBG) - gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_D8, 0x7, 3, 0); - - // Enable OMC thermal sensor controller - // This bit should be asserted 100 us after ENAD ENVR ENBG - schedule_timeout(usecs_to_jiffies(100)); - gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, - APEX_BAR2_REG_OMC0_DC, 0x1, 1, 0); + enable_thermal_sensing(gasket_dev); ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, apex_sysfs_attrs); @@ -1109,6 +1125,28 @@ remove_device: pci_disable_device(pci_dev); } +static int apex_pci_resume(struct pci_dev *pci_dev) +{ + struct apex_dev *apex_dev = pci_get_drvdata(pci_dev); + struct gasket_dev *gasket_dev; + + if (!apex_dev) { + dev_err_once(&pci_dev->dev, "NULL apex_dev\n"); + return -ENODEV; + } + gasket_dev = apex_dev->gasket_dev_ptr; + + apex_reset(gasket_dev); + program_hw_temp_warnings(apex_dev); + enable_thermal_sensing(gasket_dev); + + /* Place device in low power mode until opened */ + if (allow_power_save) + apex_enter_reset(gasket_dev); + + return 0; +} + static struct gasket_driver_desc apex_desc = { .name = "apex", .driver_version = APEX_DRIVER_VERSION, @@ -1155,6 +1193,9 @@ static struct pci_driver apex_pci_driver = { .name = "apex", .probe = apex_pci_probe, .remove = apex_pci_remove, +#ifdef CONFIG_PM_SLEEP + .resume = apex_pci_resume, +#endif .id_table = apex_pci_ids, }; From 64728f311432ef2706a7f42b36ee5e925750fdc1 Mon Sep 17 00:00:00 2001 From: Jonas Larsson Date: Thu, 11 Jun 2020 09:19:54 -0700 Subject: [PATCH 207/209] staging: gasket: fix compilation on 5.1+ https://lore.kernel.org/patchwork/patch/1039724/ Bug: 158716452 Change-Id: I3c90a6524d3859e5518b7aece5a2817ed1f3efed (cherry picked from commit 0aacc629f48459a1f8169120afc868cdb48f0593) --- gasket_page_table.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gasket_page_table.c b/gasket_page_table.c index e495658..6cd8c0b 100644 --- a/gasket_page_table.c +++ b/gasket_page_table.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include "gasket_constants.h" @@ -532,7 +533,12 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, return -EINVAL; /* Page already mapped for DMA. */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) ptes[i].dma_addr = sg_page_iter_dma_address(sg_iter); +#else + ptes[i].dma_addr = sg_page_iter_dma_address( + container_of(sg_iter, struct sg_dma_page_iter, base)); +#endif ptes[i].page = NULL; offset = 0; } else if (is_coherent(pg_tbl, host_addr)) { From 860ed615fa42ba3ab4984fca75035b72e989c226 Mon Sep 17 00:00:00 2001 From: Leonid Lobachev Date: Fri, 7 Aug 2020 15:00:59 -0700 Subject: [PATCH 208/209] Change ioremap_nocache to ioremap. ioremap defaults to ioremap_nocache since 2.6 and latter is removed in 5.6 kernel. See https://github.com/torvalds/linux/commit/4bdc0d676a643140bdf17dbf7eafedee3d496a3c Change-Id: Ifd9ff9db5937abf487d57861465cd58b4df18ac3 --- gasket_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gasket_core.c b/gasket_core.c index 16112bb..139616b 100644 --- a/gasket_core.c +++ b/gasket_core.c @@ -316,8 +316,8 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) } gasket_dev->bar_data[bar_num].virt_base = - ioremap_nocache(gasket_dev->bar_data[bar_num].phys_base, - gasket_dev->bar_data[bar_num].length_bytes); + ioremap(gasket_dev->bar_data[bar_num].phys_base, + gasket_dev->bar_data[bar_num].length_bytes); if (!gasket_dev->bar_data[bar_num].virt_base) { dev_err(gasket_dev->dev, "Cannot remap BAR %d memory region %p\n", From c59797eba687fae0075b6199ff4405fba86a4be0 Mon Sep 17 00:00:00 2001 From: Alex Van Damme Date: Mon, 12 Oct 2020 15:18:28 -0700 Subject: [PATCH 209/209] Unregister gasket IRQs on suspend - Use the existing gasket_interrupt_msix_cleanup at suspend time to remove our IRQs from the system. Without doing this, we leave 13 IRQs per device registered, which can cause issues during suspend if many devices are loaded in the system. - Restore them at resume time, using the gasket_interrupt_reinit functionality. - This allows successful suspend without any errors taking the CPUs down, and passing the multi-tpu stress test after resume. Change-Id: Ied1aca8605c0cb3b64ba591d05312d10cf45343f --- apex_driver.c | 17 +++++++++++++++++ gasket_interrupt.c | 7 +++++-- gasket_interrupt.h | 8 ++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/apex_driver.c b/apex_driver.c index 1d7b7ce..f4cf518 100644 --- a/apex_driver.c +++ b/apex_driver.c @@ -1125,6 +1125,21 @@ remove_device: pci_disable_device(pci_dev); } +static int apex_pci_suspend(struct pci_dev *pci_dev, pm_message_t state) { + struct apex_dev *apex_dev = pci_get_drvdata(pci_dev); + struct gasket_dev *gasket_dev; + + if (!apex_dev) { + dev_err_once(&pci_dev->dev, "NULL apex_dev\n"); + return -ENODEV; + } + + // Tear down MSI-x interrupts before suspending. + gasket_dev = apex_dev->gasket_dev_ptr; + gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data); + return 0; +} + static int apex_pci_resume(struct pci_dev *pci_dev) { struct apex_dev *apex_dev = pci_get_drvdata(pci_dev); @@ -1136,6 +1151,7 @@ static int apex_pci_resume(struct pci_dev *pci_dev) } gasket_dev = apex_dev->gasket_dev_ptr; + gasket_interrupt_reinit(gasket_dev); apex_reset(gasket_dev); program_hw_temp_warnings(apex_dev); enable_thermal_sensing(gasket_dev); @@ -1194,6 +1210,7 @@ static struct pci_driver apex_pci_driver = { .probe = apex_pci_probe, .remove = apex_pci_remove, #ifdef CONFIG_PM_SLEEP + .suspend = apex_pci_suspend, .resume = apex_pci_resume, #endif .id_table = apex_pci_ids, diff --git a/gasket_interrupt.c b/gasket_interrupt.c index 1e8b960..dee62e0 100644 --- a/gasket_interrupt.c +++ b/gasket_interrupt.c @@ -395,9 +395,9 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev) return 0; } +EXPORT_SYMBOL(gasket_interrupt_init); -static void -gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) +void gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) { int i; @@ -412,7 +412,9 @@ gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) pci_disable_msix(interrupt_data->pci_dev); interrupt_data->msix_configured = 0; kfree(interrupt_data->msix_entries); + interrupt_data->msix_entries = NULL; } +EXPORT_SYMBOL(gasket_interrupt_msix_cleanup); int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) { @@ -454,6 +456,7 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) return 0; } +EXPORT_SYMBOL(gasket_interrupt_reinit); /* See gasket_interrupt.h for description. */ int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev) diff --git a/gasket_interrupt.h b/gasket_interrupt.h index b17b723..048564a 100644 --- a/gasket_interrupt.h +++ b/gasket_interrupt.h @@ -45,6 +45,14 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev); */ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev); +/* + * Clean up the MSI-x subsystem. + * @interrupt_data: The interrupt data structure for this device. + * + * Performs a teardown of the MSI-x subsystem. Does not free the underlying data structures. + */ +void gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data); + /* Handle gasket interrupt processing, called from an external handler. */ void gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data,